public static Queue msFillThreadQueue = new Queue(); public static int process_count = 0; /// /// This method will fill up the mongo db instance in a queued loop /// /// public static void FillThread() { while (FillThreadRunning) { ///These are big documents if (msFillThreadQueue.Count > 100) { List documents = new List(); lock (msFillThreadQueue) { process_count += msFillThreadQueue.Count; while (msFillThreadQueue.Count > 0) { documents.Add(msFillThreadQueue.Dequeue()); } } //Commented out to see non mongo memory usage //MongoClientSettings settings = new MongoClientSettings(); //MongoCredential credential = MongoCredential.CreateMongoCRCredential("redacted", "redacted", "v"); //settings.Credentials = new MongoCredential[] { credential }; //settings.Server = new MongoServerAddress("redacted", redacted); //MongoClient client = new MongoClient(settings); //MongoServer server = client.GetServer(); //MongoDatabase database = server.GetDatabase("redacted"); //MongoCollection collection = database.GetCollection("redacted"); //collection.InsertBatch(documents); } System.Threading.Thread.Sleep(500); } //Get the last few items if (msFillThreadQueue.Count > 0) { List documents = new List(); lock (msFillThreadQueue) { while (msFillThreadQueue.Count > 0) { documents.Add(msFillThreadQueue.Dequeue()); } } //MongoClientSettings settings = new MongoClientSettings(); //MongoCredential credential = MongoCredential.CreateMongoCRCredential("redacted", "redacted", "redacted"); //settings.Credentials = new MongoCredential[] { credential }; //settings.Server = new MongoServerAddress("redacted", redacted); //MongoClient client = new MongoClient(settings); //MongoServer server = client.GetServer(); //MongoDatabase database = server.GetDatabase("redacted"); //MongoCollection collection = database.GetCollection("redacted"); //collection.InsertBatch(documents); } } public static bool FillThreadRunning = false; /// /// Will fill mongo with the data from the cache /// /// public static void FillMongo(string path) { FillThreadRunning = true; System.Threading.Tasks.Task updateTask = System.Threading.Tasks.Task.Factory.StartNew(() => { FillThread(); }); string[] files = Directory.GetFiles(path); System.Threading.Tasks.Parallel.ForEach(files, file => { if (file.EndsWith("xml")) { EDNADocument document = null; using (System.IO.FileStream fileStream = new System.IO.FileStream(file, System.IO.FileMode.Open, System.IO.FileAccess.Read)) { if (fileStream.Length > 0) { System.Xml.Serialization.XmlSerializer serializer = new System.Xml.Serialization.XmlSerializer(typeof(EDNADocument)); document = (EDNADocument)serializer.Deserialize(fileStream); } } if (document != null) { DateTime startDate = DateTime.Parse("2014-01-01"); while (startDate <= DateTime.Parse("2014-12-31")) { HelpFillMongo(document, startDate); startDate = startDate.AddDays(1); } } } }); FillThreadRunning = false; System.Threading.Tasks.Task.WaitAll(updateTask); } /// /// Will load the document from the file, fake the date and then populate it in mongo /// /// /// public static void HelpFillMongo(EDNADocument document, DateTime date) { document.Day = date.Date; document.SetKey(); foreach (Point point in document.Points) { if (point.LastKnownValue != null) { point.LastKnownValue.Time = ChangeDay(date, point.LastKnownValue.Time); } foreach (PointValue pv in point.Values) { pv.Time = ChangeDay(date, pv.Time); } } lock (msFillThreadQueue) { msFillThreadQueue.Enqueue(document.Clone()); } } public static DateTime ChangeDay(DateTime day, DateTime value) { return new DateTime(day.Year, day.Month, day.Day, value.Hour, value.Minute, value.Second, value.Millisecond); }