In [2]: %paste
import gzip
import bson
from bson import json_util
from bson import Binary
import pymongo
from pymongo import MongoClient
import time
import timeit

class Timer:
    def __init__(self, comment):
        self.comment = comment
    def __enter__(self):
        self.start = time.time()
        return self
    def __exit__(self, *args):
        self.end = time.time()
        self.interval = self.end - self.start
        print self.comment, self.interval, "s"


c = MongoClient(['cn15-ib:27119', 'cn16-ib:27119', 'localhost:27119'])
db = c.mongoose_jblackburn
db.authenticate('jblackburn', 'test')

coll = db.python_906
coll.ensure_index([('symbol', pymongo.HASHED)], cache_for=24 * 60 * 60, background=True)
coll.ensure_index([('symbol', pymongo.ASCENDING), ('sha', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True)
coll.ensure_index([('symbol', pymongo.ASCENDING),
                   ('parent', pymongo.ASCENDING),
                   ('segment', pymongo.ASCENDING)], unique=True, cache_for=24 * 60 * 60, background=True)


coll.remove()
with gzip.open('sanitized_docs.json.gz', 'r') as slow_docs:
    for i, doc in enumerate(slow_docs):
        coll.insert(json_util.loads(doc))
        if i % 100 == 0:
            print i
#        docs.append(json_util.loads(doc))
print coll.count(), " docs"


# find
parent = bson.ObjectId('5539fa60cdc41b12a4226c3f')
def find():
    with Timer("coll.find"):
        parent = coll.find_one({'symbol': 'FUT_FTL'})['parent'][0]
        list(coll.find({'symbol': 'FUT_FTL', 'parent': parent}))
print timeit.timeit(find, number=100)
## -- End pasted text --
0
100
200
232  docs
coll.find 0.388190031052 s
coll.find 0.187154054642 s
coll.find 0.289838075638 s
coll.find 0.472524881363 s
coll.find 0.267184972763 s
coll.find 0.239038944244 s
coll.find 0.241472005844 s
coll.find 0.250422000885 s
coll.find 0.24831700325 s
coll.find 0.187438964844 s
coll.find 0.464166164398 s
coll.find 0.250340938568 s
coll.find 0.185608148575 s
coll.find 0.186402082443 s
coll.find 0.916692018509 s
coll.find 0.466959953308 s
coll.find 0.242486000061 s
coll.find 0.515540122986 s
coll.find 0.179439067841 s
coll.find 0.277268886566 s
coll.find 0.18360209465 s
coll.find 0.509667158127 s
coll.find 0.257114887238 s
coll.find 0.189908027649 s
coll.find 0.481589078903 s
coll.find 0.185936927795 s
coll.find 0.236046075821 s
coll.find 0.2892100811 s
coll.find 0.180780887604 s
coll.find 0.266270160675 s
coll.find 0.181268930435 s
coll.find 0.180618047714 s
coll.find 0.267774820328 s
coll.find 0.261985063553 s
coll.find 0.488118886948 s
coll.find 0.66729092598 s
coll.find 0.252991199493 s
coll.find 0.50293302536 s
coll.find 0.225540876389 s
coll.find 0.245539903641 s
coll.find 0.255214929581 s
coll.find 0.51918888092 s
coll.find 0.188108921051 s
coll.find 0.186096906662 s
coll.find 0.280715942383 s
coll.find 0.261266946793 s
coll.find 0.187552928925 s
coll.find 0.274884939194 s
coll.find 0.194142818451 s
coll.find 0.249662876129 s
coll.find 0.181245088577 s
coll.find 0.446478128433 s
coll.find 0.257082939148 s
coll.find 0.249674081802 s
coll.find 0.711071014404 s
coll.find 0.464746952057 s
coll.find 0.467669963837 s
coll.find 0.263967990875 s
coll.find 0.445668935776 s
coll.find 0.253293037415 s
coll.find 0.18138718605 s
coll.find 0.445776939392 s
coll.find 0.171690940857 s
coll.find 0.264936923981 s
coll.find 0.466206073761 s
coll.find 0.515033006668 s
coll.find 0.267935037613 s
coll.find 0.259976148605 s
coll.find 0.262906074524 s
coll.find 0.268404006958 s
coll.find 0.268023967743 s
coll.find 0.179657936096 s
coll.find 0.17356300354 s
coll.find 0.461840867996 s
coll.find 0.254520893097 s
coll.find 0.182841062546 s
coll.find 0.18844294548 s
coll.find 0.330752134323 s
coll.find 0.186653137207 s
coll.find 0.266894817352 s
coll.find 0.179373979568 s
coll.find 0.186444997787 s
coll.find 0.250782012939 s
coll.find 0.182636022568 s
coll.find 0.703592061996 s
coll.find 0.246943950653 s
coll.find 0.181626796722 s
coll.find 0.223905086517 s
coll.find 0.184947013855 s
coll.find 0.248715877533 s
coll.find 0.26545214653 s
coll.find 0.492290973663 s
coll.find 0.242995023727 s
coll.find 0.249826908112 s
coll.find 0.250890016556 s
coll.find 0.5103328228 s
coll.find 0.248705863953 s
coll.find 0.186540126801 s
coll.find 0.239149093628 s
coll.find 0.243340015411 s
29.9709808826