Topic Modeling: Add topic evaluation scores

orangecontrib/text/widgets/owtopicmodeling.py

@@ -1,11 +1,14 @@
 import functools
+import numpy as np
 
 from AnyQt import QtGui, QtCore
 from AnyQt.QtCore import pyqtSignal, QSize
 from AnyQt.QtWidgets import (QVBoxLayout, QButtonGroup, QRadioButton,
                              QGroupBox, QTreeWidgetItem, QTreeWidget,
                              QStyleOptionViewItem, QStyledItemDelegate, QStyle)
 
+from gensim.models import CoherenceModel
+
 from Orange.widgets import settings
 from Orange.widgets import gui
 from Orange.widgets.settings import DomainContextHandler
@@ -142,6 +145,8 @@ def __init__(self):
         self.corpus = None
         self.learning_thread = None
         self.__pending_selection = self.selection
+        self.perplexity = "n/a"
+        self.coherence = "n/a"
 
         # Commit button
         gui.auto_commit(self.buttonsArea, self, 'autocommit', 'Commit', box=False)
@@ -168,6 +173,11 @@ def __init__(self):
         self.toggle_widgets()
         method_layout.addStretch()
 
+        box = gui.vBox(self.controlArea, "Topic evaluation")
+        gui.label(box, self, "Log perplexity: %(perplexity)s")
+        gui.label(box, self, "Topic coherence: %(coherence)s")
+        self.controlArea.layout().insertWidget(1, box)
+
         # Topics description
         self.topic_desc = TopicViewer()
         self.topic_desc.topicSelected.connect(self.send_topic_by_id)
@@ -231,6 +241,15 @@ def on_result(self, corpus):
                 self.__pending_selection = None
             if self.model.actual_topics != self.model.num_topics:
                 self.Warning.less_topics_found()
+            if self.model.name == "Latent Dirichlet Allocation":
+                bound = self.model.model.log_perplexity(
+                    corpus.ngrams_corpus)
+                self.perplexity = "{:.5f}".format(np.exp2(-bound))
+            cm = CoherenceModel(model=self.model.model,
+                                texts=corpus.tokens, corpus=corpus,
+                                coherence='c_v')
+            coherence = cm.get_coherence()
+            self.coherence = "{:.5f}".format(coherence)
             self.Outputs.all_topics.send(self.model.get_all_topics_table())
 
     @learning_task.callback
@@ -381,4 +400,3 @@ def sizeHint(self, option, index):
     widget.set_data(Corpus.from_file('deerwester'))
     widget.show()
     app.exec()
-    widget.saveSettings()

orangecontrib/text/widgets/tests/test_owtopicmodeling.py

@@ -66,6 +66,26 @@ def until(widget=self.widget):
         self.assertEqual(output.metas.shape[1],
                          self.widget.corpus.metas.shape[1] + 1)
 
+    def test_topic_evaluation(self):
+        def until(widget=self.widget):
+            return bool(self.get_output(widget.Outputs.selected_topic,
+                                        widget=widget))
+
+        self.send_signal(self.widget.Inputs.corpus, self.corpus)
+        self.process_events(until)
+
+        # test LSI
+        self.assertEqual(self.widget.perplexity, "n/a")
+        self.assertTrue(self.widget.coherence)
+
+        # test LDA, which is the only one with log perplexity
+        self.widget.method_index = 1
+        self.widget.commit()
+        self.process_events(until)
+
+        self.assertNotEqual(self.widget.perplexity, "n/a")
+        self.assertTrue(self.widget.coherence)
+
 
 if __name__ == "__main__":
     unittest.main()