SqliteDict memory issue #48

from sqlitedict import SqliteDict
from gensim.models import TfidfModel, LsiModel
from gensim.models.lsimodel import Projection
from gensim.utils import grouper
from gensim import matutils

s = SqliteDict('documents.sqlite')  # contains namedtuples with field `tokens` containing unicode token tuple
tfidf = TfidfModel.load('tfidf_model.pkl')  # trained on a dictionary built from `s`


class SegmentCorpus(object):
    def __init__(self, storage, built_dictionary):
        self.storage = storage
        self.inner_dictionary = built_dictionary

    def __iter__(self):
        for segment in self.storage.itervalues():
            yield self.inner_dictionary.doc2bow(segment.tokens)


corpus_tfidf = tfidf[SegmentCorpus(s, tfidf.id2word)]
for i in grouper(corpus_tfidf, 5000):
    nnz = sum(len(doc) for doc in i)
    job = matutils.corpus2csc(i, num_docs=len(i), num_nnz=nnz, num_terms=len(tfidf.id2word))
    Projection(len(tfidf.id2word), 100, job, extra_dims=100, power_iters=1)