In [4]: %paste
import gzip
import bson
from bson import json_util
from bson import Binary
import pymongo
from pymongo import MongoClient
import time
import timeit

class Timer:
    def __init__(self, comment):
        self.comment = comment
    def __enter__(self):
        self.start = time.time()
        return self
    def __exit__(self, *args):
        self.end = time.time()
        self.interval = self.end - self.start
        print self.comment, self.interval, "s"


c = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119'])
db = c.mongoose_jblackburn
db.authenticate('jblackburn', 'test')

coll = db.python_906
coll.ensure_index([('symbol', pymongo.HASHED)], cache_for=24 * 60 * 60, background=True)
coll.ensure_index([('symbol', pymongo.ASCENDING), ('sha', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True)
coll.ensure_index([('symbol', pymongo.ASCENDING),
                   ('parent', pymongo.ASCENDING),
                   ('segment', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True)


coll.remove()
with gzip.open('sanitized_docs.json.gz', 'r') as slow_docs:
    for i, doc in enumerate(slow_docs):
        coll.insert(json_util.loads(doc))
        if i % 100 == 0:
            print i
#        docs.append(json_util.loads(doc))
print coll.count(), " docs"


# find
parent = bson.ObjectId('5539fa60cdc41b12a4226c3f')
def find():
    with Timer("coll.find"):
        parent = coll.find_one({'symbol': 'FUT_FTL'})['parent'][0]
        list(coll.find({'symbol': 'FUT_FTL', 'parent': parent}))
print timeit.timeit(find, number=100)
## -- End pasted text --
0
100
200
232  docs
coll.find 0.175731897354 s
coll.find 0.640354156494 s
coll.find 0.181133031845 s
coll.find 0.178170919418 s
coll.find 0.381839990616 s
coll.find 0.173588037491 s
coll.find 0.185986995697 s
coll.find 0.184150934219 s
coll.find 0.175131797791 s
coll.find 0.176074028015 s
coll.find 0.186100006104 s
coll.find 0.180463075638 s
coll.find 0.184152841568 s
coll.find 0.188079118729 s
coll.find 0.181241989136 s
coll.find 0.423246860504 s
coll.find 0.178884029388 s
coll.find 0.176948070526 s
coll.find 0.185127019882 s
coll.find 0.180176973343 s
coll.find 0.181604862213 s
coll.find 0.425833940506 s
coll.find 0.191309928894 s
coll.find 0.181257009506 s
coll.find 0.172424077988 s
coll.find 0.176941871643 s
coll.find 0.182662010193 s
coll.find 0.163249015808 s
coll.find 0.418696165085 s
coll.find 0.187355041504 s
coll.find 0.439073085785 s
coll.find 0.172821998596 s
coll.find 0.181853055954 s
coll.find 0.19086098671 s
coll.find 0.188935995102 s
coll.find 0.182960987091 s
coll.find 0.182603120804 s
coll.find 0.195058107376 s
coll.find 0.639644861221 s
coll.find 0.185006141663 s
coll.find 0.181931972504 s
coll.find 0.437393903732 s
coll.find 0.185714006424 s
coll.find 0.178777933121 s
coll.find 0.176234006882 s
coll.find 0.181764125824 s
coll.find 0.182502985001 s
coll.find 0.173329114914 s
coll.find 0.184699058533 s
coll.find 0.176037073135 s
coll.find 0.164447069168 s
coll.find 0.161071062088 s
coll.find 0.17201089859 s
coll.find 0.178879976273 s
coll.find 0.173259019852 s
coll.find 0.172425031662 s
coll.find 0.178198099136 s
coll.find 0.174580097198 s
coll.find 0.18286895752 s
coll.find 0.176783084869 s
coll.find 0.183357954025 s
coll.find 0.385584831238 s
coll.find 0.17539191246 s
coll.find 0.18280005455 s
coll.find 0.18537902832 s
coll.find 0.43415594101 s
coll.find 0.187131881714 s
coll.find 0.18043589592 s
coll.find 0.184289932251 s
coll.find 0.430021047592 s
coll.find 0.185068845749 s
coll.find 0.190948009491 s
coll.find 0.431650161743 s
coll.find 0.180830001831 s
coll.find 0.183779001236 s
coll.find 0.187206029892 s
coll.find 0.180133104324 s
coll.find 0.178870916367 s
coll.find 0.183619976044 s
coll.find 0.190557956696 s
coll.find 0.183173894882 s
coll.find 0.191443920135 s
coll.find 0.190629005432 s
coll.find 0.192770004272 s
coll.find 0.173548936844 s
coll.find 0.172280073166 s
coll.find 0.18714094162 s
coll.find 0.435049057007 s
coll.find 0.182077169418 s
coll.find 0.437000989914 s
coll.find 0.179808139801 s
coll.find 0.183572053909 s
coll.find 0.189491033554 s
coll.find 0.180776119232 s
coll.find 0.184168100357 s
coll.find 0.165666818619 s
coll.find 0.428196907043 s
coll.find 0.179425001144 s
coll.find 0.18831205368 s
coll.find 0.177426815033 s
22.1772301197