In [42]: %paste import gzip import bson from bson import json_util from bson import Binary import pymongo from pymongo import MongoClient import time import timeit import time from pymongo import MongoClient try: from pymongo.server_selectors import any_server_selector except: pass class Timer: def __init__(self, comment): self.comment = comment def __enter__(self): self.start = time.time() return self def __exit__(self, *args): self.end = time.time() self.interval = self.end - self.start print self.comment, self.interval, "s" # client = MongoClient(['cn54-ib:27119', 'cn55-ib:27119', 'cn56-ib:27119', 'localhost:27119'], localThresholdMS=0.0000000001) # client = MongoClient(['cn54-ib:27119', 'cn55-ib:27119', 'cn56-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock'], localThresholdMS=0.0000000001) # client = MongoClient(['cn54-ib:27119', 'cn55-ib:27119', 'cn56-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock'], localThresholdMS=0.0000000001) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'cn90-ib:27119', 'mongodb:///tmp/mongodb-27119.sock']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'cn90-ib:27119', 'mongodb:///tmp/mongodb-27119.sock'], localThresholdMS=0.0000000001) client = MongoClient(['cn15-ib:27119'], localThresholdMS=0.0000000001) # c = MongoClient(['localhost:27119']) db = client.mongoose_jblackburn db.authenticate('jblackburn', 'test') start = time.time() # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119']) for _ in range(10): servers = client._topology.select_servers(any_server_selector) print('%.2f seconds' % (time.time() - start)) for s in servers: print('%s\t%.8f' % ( s.description.address, s.description.round_trip_time)) time.sleep(10) coll = db.python_906 # coll.ensure_index([('symbol', pymongo.HASHED)], cache_for=24 * 60 * 60, background=True) # coll.ensure_index([('symbol', pymongo.ASCENDING), ('sha', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True) # coll.ensure_index([('symbol', pymongo.ASCENDING), # ('parent', pymongo.ASCENDING), # ('segment', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True) # # # coll.remove() # with gzip.open('sanitized_docs.json.gz', 'r') as slow_docs: # for i, doc in enumerate(slow_docs): # coll.insert(json_util.loads(doc)) # if i % 100 == 0: # print i # # docs.append(json_util.loads(doc)) # print coll.count(), " docs" # find parent = bson.ObjectId('5539fa60cdc41b12a4226c3f') def find(): with Timer("coll.find"): parent = coll.find_one({'symbol': 'FUT_FTL'})['parent'][0] list(coll.find({'symbol': 'FUT_FTL', 'parent': parent})) for s in servers: print('%s\t%.8f' % ( s.description.address, s.description.round_trip_time)) print timeit.timeit(find, number=100) ## -- End pasted text -- 0.00 seconds ('cn15-ib', 27119) 0.00033689 10.00 seconds ('cn15-ib', 27119) 0.00031590 20.00 seconds ('cn15-ib', 27119) 0.00031090 30.00 seconds ('cn15-ib', 27119) 0.00031514 40.00 seconds ('cn15-ib', 27119) 0.00031429 50.00 seconds ('cn15-ib', 27119) 0.00030460 60.00 seconds ('cn15-ib', 27119) 0.00030248 70.00 seconds ('cn15-ib', 27119) 0.00029720 80.00 seconds ('cn15-ib', 27119) 0.00031734 90.00 seconds ('cn15-ib', 27119) 0.00030866 ('cn15-ib', 27119) 0.00030096 coll.find 0.547419071198 s ('cn15-ib', 27119) 0.00030096 coll.find 0.255429029465 s ('cn15-ib', 27119) 0.00030096 coll.find 0.300577163696 s ('cn15-ib', 27119) 0.00029956 coll.find 0.455651044846 s ('cn15-ib', 27119) 0.00029956 coll.find 0.257000207901 s ('cn15-ib', 27119) 0.00029956 coll.find 0.236135959625 s ('cn15-ib', 27119) 0.00029956 coll.find 0.450905799866 s ('cn15-ib', 27119) 0.00029956 coll.find 0.253557920456 s ('cn15-ib', 27119) 0.00029956 coll.find 0.259983062744 s ('cn15-ib', 27119) 0.00029956 coll.find 0.255973815918 s ('cn15-ib', 27119) 0.00029956 coll.find 0.295587062836 s ('cn15-ib', 27119) 0.00029956 coll.find 0.246134996414 s ('cn15-ib', 27119) 0.00029956 coll.find 0.249112844467 s ('cn15-ib', 27119) 0.00029956 coll.find 0.253386974335 s ('cn15-ib', 27119) 0.00029956 coll.find 0.510786056519 s ('cn15-ib', 27119) 0.00029956 coll.find 0.273990869522 s ('cn15-ib', 27119) 0.00029956 coll.find 0.246709108353 s ('cn15-ib', 27119) 0.00029956 coll.find 0.252377033234 s ('cn15-ib', 27119) 0.00029956 coll.find 0.226224899292 s ('cn15-ib', 27119) 0.00029956 coll.find 0.292716026306 s ('cn15-ib', 27119) 0.00029956 coll.find 0.508374929428 s ('cn15-ib', 27119) 0.00029956 coll.find 0.256308078766 s ('cn15-ib', 27119) 0.00029956 coll.find 0.310636997223 s ('cn15-ib', 27119) 0.00029956 coll.find 0.258352994919 s ('cn15-ib', 27119) 0.00029956 coll.find 0.263375997543 s ('cn15-ib', 27119) 0.00029956 coll.find 0.496062994003 s ('cn15-ib', 27119) 0.00029956 coll.find 0.483526945114 s ('cn15-ib', 27119) 0.00029956 coll.find 0.324764966965 s ('cn15-ib', 27119) 0.00029956 coll.find 0.291027784348 s ('cn15-ib', 27119) 0.00029956 coll.find 0.260031938553 s ('cn15-ib', 27119) 0.00029956 coll.find 0.242902040482 s ('cn15-ib', 27119) 0.00029956 coll.find 0.254688024521 s ('cn15-ib', 27119) 0.00029956 coll.find 0.261048793793 s ('cn15-ib', 27119) 0.00029956 coll.find 0.263702869415 s ('cn15-ib', 27119) 0.00029956 coll.find 0.258681058884 s ('cn15-ib', 27119) 0.00029956 coll.find 0.262366056442 s ('cn15-ib', 27119) 0.00036405 coll.find 0.267282962799 s ('cn15-ib', 27119) 0.00036405 coll.find 0.243340969086 s ('cn15-ib', 27119) 0.00036405 coll.find 0.262966871262 s ('cn15-ib', 27119) 0.00036405 coll.find 0.539618015289 s ('cn15-ib', 27119) 0.00036405 coll.find 0.33003115654 s ('cn15-ib', 27119) 0.00036405 coll.find 0.255190134048 s ('cn15-ib', 27119) 0.00036405 coll.find 0.25760602951 s ('cn15-ib', 27119) 0.00036405 coll.find 0.705476999283 s ('cn15-ib', 27119) 0.00036405 coll.find 0.280886173248 s ('cn15-ib', 27119) 0.00036405 coll.find 0.284911870956 s ('cn15-ib', 27119) 0.00036405 coll.find 0.2614839077 s ('cn15-ib', 27119) 0.00036405 coll.find 0.265290975571 s ('cn15-ib', 27119) 0.00036405 coll.find 0.233438014984 s ('cn15-ib', 27119) 0.00036405 coll.find 0.336077928543 s ('cn15-ib', 27119) 0.00036405 coll.find 0.465256929398 s ('cn15-ib', 27119) 0.00036405 coll.find 0.258268117905 s ('cn15-ib', 27119) 0.00036405 coll.find 0.261609077454 s ('cn15-ib', 27119) 0.00036405 coll.find 0.240370035172 s ('cn15-ib', 27119) 0.00036405 coll.find 0.220443964005 s ('cn15-ib', 27119) 0.00036405 coll.find 0.243662118912 s ('cn15-ib', 27119) 0.00036405 coll.find 0.29226398468 s ('cn15-ib', 27119) 0.00036405 coll.find 0.240175008774 s ('cn15-ib', 27119) 0.00036405 coll.find 0.235090970993 s ('cn15-ib', 27119) 0.00036405 coll.find 0.518086194992 s ('cn15-ib', 27119) 0.00036405 coll.find 0.242809057236 s ('cn15-ib', 27119) 0.00036405 coll.find 0.246585130692 s ('cn15-ib', 27119) 0.00036405 coll.find 0.275717973709 s ('cn15-ib', 27119) 0.00036405 coll.find 0.259817838669 s ('cn15-ib', 27119) 0.00036405 coll.find 0.308300018311 s ('cn15-ib', 27119) 0.00036405 coll.find 0.456199884415 s ('cn15-ib', 27119) 0.00036405 coll.find 0.294481992722 s ('cn15-ib', 27119) 0.00036405 coll.find 0.353892803192 s ('cn15-ib', 27119) 0.00033363 coll.find 0.326852083206 s ('cn15-ib', 27119) 0.00033363 coll.find 0.237324953079 s ('cn15-ib', 27119) 0.00033363 coll.find 0.266892910004 s ('cn15-ib', 27119) 0.00033363 coll.find 0.291312932968 s ('cn15-ib', 27119) 0.00033363 coll.find 0.230473041534 s ('cn15-ib', 27119) 0.00033363 coll.find 0.261479854584 s ('cn15-ib', 27119) 0.00033363 coll.find 0.297731876373 s ('cn15-ib', 27119) 0.00033363 coll.find 0.278239965439 s ('cn15-ib', 27119) 0.00033363 coll.find 0.26612496376 s ('cn15-ib', 27119) 0.00033363 coll.find 0.480875015259 s ('cn15-ib', 27119) 0.00033363 coll.find 0.319508075714 s ('cn15-ib', 27119) 0.00033363 coll.find 0.252270936966 s ('cn15-ib', 27119) 0.00033363 coll.find 0.519060850143 s ('cn15-ib', 27119) 0.00033363 coll.find 0.263033151627 s ('cn15-ib', 27119) 0.00033363 coll.find 0.239325046539 s ('cn15-ib', 27119) 0.00033363 coll.find 0.261914968491 s ('cn15-ib', 27119) 0.00033363 coll.find 0.281162977219 s ('cn15-ib', 27119) 0.00033363 coll.find 0.520972013474 s ('cn15-ib', 27119) 0.00033363 coll.find 0.237818002701 s ('cn15-ib', 27119) 0.00033363 coll.find 0.251415014267 s ('cn15-ib', 27119) 0.00033363 coll.find 0.264401912689 s ('cn15-ib', 27119) 0.00033363 coll.find 0.266374826431 s ('cn15-ib', 27119) 0.00033363 coll.find 0.302595853806 s ('cn15-ib', 27119) 0.00033363 coll.find 0.48682808876 s ('cn15-ib', 27119) 0.00033363 coll.find 0.258767127991 s ('cn15-ib', 27119) 0.00033363 coll.find 0.256011009216 s ('cn15-ib', 27119) 0.00033363 coll.find 0.274946928024 s ('cn15-ib', 27119) 0.00033363 coll.find 0.506550073624 s ('cn15-ib', 27119) 0.00033363 coll.find 0.497822999954 s ('cn15-ib', 27119) 0.00033363 coll.find 0.232534885406 s ('cn15-ib', 27119) 0.00033363 coll.find 0.511307001114 s ('cn15-ib', 27119) 0.00032532 coll.find 0.691142082214 s 31.694947958 In [43]: %paste import gzip import bson from bson import json_util from bson import Binary import pymongo from pymongo import MongoClient import time import timeit import time from pymongo import MongoClient try: from pymongo.server_selectors import any_server_selector except: pass class Timer: def __init__(self, comment): self.comment = comment def __enter__(self): self.start = time.time() return self def __exit__(self, *args): self.end = time.time() self.interval = self.end - self.start print self.comment, self.interval, "s" # client = MongoClient(['cn54-ib:27119', 'cn55-ib:27119', 'cn56-ib:27119', 'localhost:27119'], localThresholdMS=0.0000000001) # client = MongoClient(['cn54-ib:27119', 'cn55-ib:27119', 'cn56-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock'], localThresholdMS=0.0000000001) # client = MongoClient(['cn54-ib:27119', 'cn55-ib:27119', 'cn56-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'mongodb:///tmp/mongodb-27119.sock'], localThresholdMS=0.0000000001) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'cn90-ib:27119', 'mongodb:///tmp/mongodb-27119.sock']) # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119', 'cn90-ib:27119', 'mongodb:///tmp/mongodb-27119.sock'], localThresholdMS=0.0000000001) client = MongoClient(['cn15-ib:27119'], localThresholdMS=0.0000000001) client = MongoClient(['localhost:27119'], localThresholdMS=0.0000000001) # c = MongoClient(['localhost:27119']) db = client.mongoose_jblackburn db.authenticate('jblackburn', 'test') start = time.time() # client = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119']) for _ in range(10): servers = client._topology.select_servers(any_server_selector) print('%.2f seconds' % (time.time() - start)) for s in servers: print('%s\t%.8f' % ( s.description.address, s.description.round_trip_time)) time.sleep(10) coll = db.python_906 # coll.ensure_index([('symbol', pymongo.HASHED)], cache_for=24 * 60 * 60, background=True) # coll.ensure_index([('symbol', pymongo.ASCENDING), ('sha', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True) # coll.ensure_index([('symbol', pymongo.ASCENDING), # ('parent', pymongo.ASCENDING), # ('segment', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True) # # # coll.remove() # with gzip.open('sanitized_docs.json.gz', 'r') as slow_docs: # for i, doc in enumerate(slow_docs): # coll.insert(json_util.loads(doc)) # if i % 100 == 0: # print i # # docs.append(json_util.loads(doc)) # print coll.count(), " docs" # find parent = bson.ObjectId('5539fa60cdc41b12a4226c3f') def find(): with Timer("coll.find"): parent = coll.find_one({'symbol': 'FUT_FTL'})['parent'][0] list(coll.find({'symbol': 'FUT_FTL', 'parent': parent})) for s in servers: print('%s\t%.8f' % ( s.description.address, s.description.round_trip_time)) print timeit.timeit(find, number=100) ## -- End pasted text -- 0.00 seconds ('localhost', 27119) 0.00021696 10.00 seconds ('localhost', 27119) 0.00022540 20.00 seconds ('localhost', 27119) 0.00024136 30.00 seconds ('localhost', 27119) 0.00023629 40.00 seconds ('localhost', 27119) 0.00025865 50.00 seconds ('localhost', 27119) 0.00027348 60.00 seconds ('localhost', 27119) 0.00027057 70.00 seconds ('localhost', 27119) 0.00028507 80.00 seconds ('localhost', 27119) 0.00029844 90.00 seconds ('localhost', 27119) 0.00030975 ('localhost', 27119) 0.00030941 coll.find 0.174930095673 s ('localhost', 27119) 0.00030941 coll.find 0.175271987915 s ('localhost', 27119) 0.00030941 coll.find 0.172609806061 s ('localhost', 27119) 0.00030941 coll.find 0.175622940063 s ('localhost', 27119) 0.00030941 coll.find 0.422159910202 s ('localhost', 27119) 0.00029116 coll.find 0.17512011528 s ('localhost', 27119) 0.00029116 coll.find 0.179440975189 s ('localhost', 27119) 0.00029116 coll.find 0.17667388916 s ('localhost', 27119) 0.00029116 coll.find 0.180283069611 s ('localhost', 27119) 0.00029116 coll.find 0.178300142288 s ('localhost', 27119) 0.00029116 coll.find 0.17721581459 s ('localhost', 27119) 0.00029116 coll.find 0.17867398262 s ('localhost', 27119) 0.00029116 coll.find 0.173584938049 s ('localhost', 27119) 0.00029116 coll.find 0.176458835602 s ('localhost', 27119) 0.00029116 coll.find 0.173061847687 s ('localhost', 27119) 0.00029116 coll.find 0.152959108353 s ('localhost', 27119) 0.00029116 coll.find 0.153158187866 s ('localhost', 27119) 0.00029116 coll.find 0.174386978149 s ('localhost', 27119) 0.00029116 coll.find 0.172518968582 s ('localhost', 27119) 0.00029116 coll.find 0.176205873489 s ('localhost', 27119) 0.00029116 coll.find 0.17534995079 s ('localhost', 27119) 0.00029116 coll.find 0.174255132675 s ('localhost', 27119) 0.00029116 coll.find 0.174632072449 s ('localhost', 27119) 0.00029116 coll.find 0.423364162445 s ('localhost', 27119) 0.00029116 coll.find 0.175992012024 s ('localhost', 27119) 0.00029116 coll.find 0.428103923798 s ('localhost', 27119) 0.00029116 coll.find 0.186229944229 s ('localhost', 27119) 0.00029116 coll.find 0.215949058533 s ('localhost', 27119) 0.00029116 coll.find 0.179030895233 s ('localhost', 27119) 0.00029116 coll.find 0.170621871948 s ('localhost', 27119) 0.00029116 coll.find 0.171828985214 s ('localhost', 27119) 0.00029116 coll.find 0.168914079666 s ('localhost', 27119) 0.00029116 coll.find 0.375874996185 s ('localhost', 27119) 0.00029116 coll.find 0.171694993973 s ('localhost', 27119) 0.00029116 coll.find 0.182870149612 s ('localhost', 27119) 0.00029116 coll.find 0.178462982178 s ('localhost', 27119) 0.00029116 coll.find 0.183345079422 s ('localhost', 27119) 0.00029116 coll.find 0.184890031815 s ('localhost', 27119) 0.00029116 coll.find 0.416633844376 s ('localhost', 27119) 0.00029116 coll.find 0.174984931946 s ('localhost', 27119) 0.00029116 coll.find 0.178724050522 s ('localhost', 27119) 0.00029116 coll.find 0.179854154587 s ('localhost', 27119) 0.00029116 coll.find 0.171534061432 s ('localhost', 27119) 0.00029116 coll.find 0.178233146667 s ('localhost', 27119) 0.00029116 coll.find 0.17477107048 s ('localhost', 27119) 0.00029116 coll.find 0.174298048019 s ('localhost', 27119) 0.00029116 coll.find 0.179705858231 s ('localhost', 27119) 0.00029116 coll.find 0.177129983902 s ('localhost', 27119) 0.00029116 coll.find 0.178988933563 s ('localhost', 27119) 0.00029116 coll.find 0.172643899918 s ('localhost', 27119) 0.00029116 coll.find 0.417639970779 s ('localhost', 27119) 0.00029116 coll.find 0.172358989716 s ('localhost', 27119) 0.00029116 coll.find 0.17932510376 s ('localhost', 27119) 0.00029116 coll.find 0.177686929703 s ('localhost', 27119) 0.00029116 coll.find 0.184267997742 s ('localhost', 27119) 0.00028471 coll.find 0.185803890228 s ('localhost', 27119) 0.00028471 coll.find 0.185135126114 s ('localhost', 27119) 0.00028471 coll.find 0.181694030762 s ('localhost', 27119) 0.00028471 coll.find 0.176942110062 s ('localhost', 27119) 0.00028471 coll.find 0.184819936752 s ('localhost', 27119) 0.00028471 coll.find 0.429303884506 s ('localhost', 27119) 0.00028471 coll.find 0.175657987595 s ('localhost', 27119) 0.00028471 coll.find 0.181662082672 s ('localhost', 27119) 0.00028471 coll.find 0.181688070297 s ('localhost', 27119) 0.00028471 coll.find 0.180426120758 s ('localhost', 27119) 0.00028471 coll.find 0.180731058121 s ('localhost', 27119) 0.00028471 coll.find 0.180938005447 s ('localhost', 27119) 0.00028471 coll.find 0.185922861099 s ('localhost', 27119) 0.00028471 coll.find 0.181566953659 s ('localhost', 27119) 0.00028471 coll.find 0.181578874588 s ('localhost', 27119) 0.00028471 coll.find 0.182821989059 s ('localhost', 27119) 0.00028471 coll.find 0.182284116745 s ('localhost', 27119) 0.00028471 coll.find 0.433609008789 s ('localhost', 27119) 0.00028471 coll.find 0.673919916153 s ('localhost', 27119) 0.00028471 coll.find 0.176927089691 s ('localhost', 27119) 0.00028471 coll.find 0.17842912674 s ('localhost', 27119) 0.00028471 coll.find 0.17518901825 s ('localhost', 27119) 0.00028471 coll.find 0.178141832352 s ('localhost', 27119) 0.00028471 coll.find 0.176443815231 s ('localhost', 27119) 0.00028471 coll.find 0.1772108078 s ('localhost', 27119) 0.00028471 coll.find 0.178979873657 s ('localhost', 27119) 0.00028471 coll.find 0.181429862976 s ('localhost', 27119) 0.00028471 coll.find 0.185159921646 s ('localhost', 27119) 0.00028471 coll.find 0.174456119537 s ('localhost', 27119) 0.00028471 coll.find 0.156309127808 s ('localhost', 27119) 0.00028471 coll.find 0.180112838745 s ('localhost', 27119) 0.00028471 coll.find 0.180698871613 s ('localhost', 27119) 0.00028471 coll.find 0.182636022568 s ('localhost', 27119) 0.00028471 coll.find 0.17866897583 s ('localhost', 27119) 0.00028471 coll.find 0.176796913147 s ('localhost', 27119) 0.00028471 coll.find 0.178654193878 s ('localhost', 27119) 0.00028471 coll.find 0.634906053543 s ('localhost', 27119) 0.00028471 coll.find 0.173005104065 s ('localhost', 27119) 0.00028471 coll.find 0.180157899857 s ('localhost', 27119) 0.00028471 coll.find 0.413754940033 s ('localhost', 27119) 0.00028471 coll.find 0.172808170319 s ('localhost', 27119) 0.00028471 coll.find 0.17561006546 s ('localhost', 27119) 0.00028471 coll.find 0.182142019272 s ('localhost', 27119) 0.00028471 coll.find 0.420537948608 s ('localhost', 27119) 0.00028471 coll.find 0.175786018372 s 21.1288309097 In [44]: