add tfidf visual

Author: morvanzhou

tf_idf.py

@@ -1,6 +1,7 @@
 import numpy as np
 from collections import Counter
 import itertools
+from visual import show_tfidf
 
 docs = [
     "it is a good day, I like to stay here",
@@ -133,4 +134,6 @@ def get_keywords(n=2):
 q = "I get a coffee cup"
 scores = docs_score(q)
 d_ids = scores.argsort()[-3:][::-1]
-print("\ntop 3 docs for '{}':\n{}".format(q, [docs[i] for i in d_ids]))
+print("\ntop 3 docs for '{}':\n{}".format(q, [docs[i] for i in d_ids]))
+
+show_tfidf(tf_idf.T, [i2v[i] for i in range(len(i2v))], "tfidf_matrix")

tf_idf_sklearn.py

@@ -1,5 +1,6 @@
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
+from visual import show_tfidf
 
 
 docs = [
@@ -32,3 +33,6 @@
 res = res.ravel().argsort()[-3:]
 print("\ntop 3 docs for '{}':\n{}".format(q, [docs[i] for i in res[::-1]]))
 
+
+i2v = {i: v for v, i in vectorizer.vocabulary_.items()}
+show_tfidf(tf_idf.todense(), [i2v[i] for i in range(len(i2v))], "tfidf_sklearn_matrix")

visual.py

@@ -6,6 +6,15 @@
 import utils
 
 
+def show_tfidf(tfidf, vocb, filename):
+    # [n_vocab, n_doc]
+    plt.imshow(tfidf, cmap="YlGn", vmin=tfidf.min(), vmax=tfidf.max())
+    plt.xticks(np.arange(tfidf.shape[1]), vocb, fontsize=6, rotation=90)
+    plt.yticks(np.arange(tfidf.shape[0]), np.arange(1, tfidf.shape[1]+1), fontsize=6)
+    plt.tight_layout()
+    plt.savefig("./visual/results/%s.png" % filename, format="png", dpi=500)
+    plt.show()
+
 def show_w2v_word_embedding(model, data: utils.Dataset, path):
     word_emb = model.embeddings.get_weights()[0]
     for i in range(data.num_word):