Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Topic Modeling: Add topic evaluation scores #687

Merged
merged 3 commits into from
Aug 13, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 60 additions & 35 deletions orangecontrib/text/widgets/owtopicmodeling.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
import functools
from typing import Any

import numpy as np

from AnyQt import QtGui, QtCore
from AnyQt.QtCore import pyqtSignal, QSize
from AnyQt.QtWidgets import (QVBoxLayout, QButtonGroup, QRadioButton,
QGroupBox, QTreeWidgetItem, QTreeWidget,
QStyleOptionViewItem, QStyledItemDelegate, QStyle)
from Orange.widgets.utils.concurrent import TaskState, ConcurrentWidgetMixin

from gensim.models import CoherenceModel

from Orange.widgets import settings
from Orange.widgets import gui
Expand All @@ -13,7 +19,7 @@
from Orange.data import Table
from orangecontrib.text.corpus import Corpus
from orangecontrib.text.topics import Topic, LdaWrapper, HdpWrapper, LsiWrapper
from orangecontrib.text.widgets.utils.concurrent import asynchronous
from orangecontrib.text.topics.topics import GensimWrapper


class TopicWidget(gui.OWComponent, QGroupBox):
Expand Down Expand Up @@ -96,7 +102,16 @@ def wrapper(self, *args, **kwargs):
return decorator


class OWTopicModeling(OWWidget):
def _run(corpus: Corpus, model: GensimWrapper, state: TaskState):
def callback(i: float):
state.set_progress_value(i * 100)
if state.is_interruption_requested():
raise Exception

return model.fit_transform(corpus.copy(), chunk_number=100, on_progress=callback)


class OWTopicModeling(OWWidget, ConcurrentWidgetMixin):
name = "Topic Modelling"
description = "Uncover the hidden thematic structure in a corpus."
icon = "icons/TopicModeling.svg"
Expand Down Expand Up @@ -137,11 +152,18 @@ class Outputs:
class Warning(OWWidget.Warning):
less_topics_found = Msg('Less topics found than requested.')

class Error(OWWidget.Error):
unexpected_error = Msg("{}")

def __init__(self):
super().__init__()
ConcurrentWidgetMixin.__init__(self)

self.corpus = None
self.learning_thread = None
self.__pending_selection = self.selection
self.perplexity = "n/a"
self.coherence = "n/a"

# Commit button
gui.auto_commit(self.buttonsArea, self, 'autocommit', 'Commit', box=False)
Expand All @@ -168,6 +190,11 @@ def __init__(self):
self.toggle_widgets()
method_layout.addStretch()

box = gui.vBox(self.controlArea, "Topic evaluation")
gui.label(box, self, "Log perplexity: %(perplexity)s")
gui.label(box, self, "Topic coherence: %(coherence)s")
self.controlArea.layout().insertWidget(1, box)

# Topics description
self.topic_desc = TopicViewer()
self.topic_desc.topicSelected.connect(self.send_topic_by_id)
Expand Down Expand Up @@ -199,43 +226,42 @@ def toggle_widgets(self):
widget.setVisible(i == self.method_index)

def apply(self):
self.learning_task.stop()
self.topic_desc.clear()
if self.corpus is not None:
self.learning_task()
self.Warning.less_topics_found.clear()
self.start(_run, self.corpus, self.model)
else:
self.on_result(None)

@asynchronous
def learning_task(self):
return self.model.fit_transform(self.corpus.copy(), chunk_number=100,
on_progress=self.on_progress)

@learning_task.on_start
def on_start(self):
self.Warning.less_topics_found.clear()
self.progressBarInit()
self.topic_desc.clear()

@learning_task.on_result
def on_result(self, corpus):
self.progressBarFinished()
self.Outputs.corpus.send(corpus)
if corpus is None:
self.topic_desc.clear()
self.Outputs.corpus.send(None)
self.Outputs.selected_topic.send(None)
self.Outputs.all_topics.send(None)
else:
self.topic_desc.show_model(self.model)
if self.__pending_selection:
self.topic_desc.select(self.__pending_selection)
self.__pending_selection = None
if self.model.actual_topics != self.model.num_topics:
self.Warning.less_topics_found()
self.Outputs.all_topics.send(self.model.get_all_topics_table())

@learning_task.callback
def on_progress(self, p):
self.progressBarSet(100 * p)

def on_done(self, corpus):
self.Outputs.corpus.send(corpus)
self.topic_desc.show_model(self.model)
if self.__pending_selection:
self.topic_desc.select(self.__pending_selection)
self.__pending_selection = None

if self.model.actual_topics != self.model.num_topics:
self.Warning.less_topics_found()

if self.model.name == "Latent Dirichlet Allocation":
bound = self.model.model.log_perplexity(corpus.ngrams_corpus)
self.perplexity = "{:.5f}".format(np.exp2(-bound))
cm = CoherenceModel(
model=self.model.model, texts=corpus.tokens, corpus=corpus, coherence="c_v"
)
coherence = cm.get_coherence()
self.coherence = "{:.5f}".format(coherence)

self.Outputs.all_topics.send(self.model.get_all_topics_table())

def on_exception(self, ex: Exception):
self.Error.unexpected_error(str(ex))

def on_partial_result(self, result: Any) -> None:
pass

def send_report(self):
self.report_items(*self.widgets[self.method_index].report_model())
Expand Down Expand Up @@ -381,4 +407,3 @@ def sizeHint(self, option, index):
widget.set_data(Corpus.from_file('deerwester'))
widget.show()
app.exec()
widget.saveSettings()
41 changes: 22 additions & 19 deletions orangecontrib/text/widgets/tests/test_owtopicmodeling.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,22 @@


class TestTopicModeling(WidgetTest):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.corpus = Corpus.from_file('deerwester')

def setUp(self):
self.corpus = Corpus.from_file("deerwester")
self.widget = self.create_widget(OWTopicModeling)

def test_data(self):
def until():
return bool(self.get_output(self.widget.Outputs.selected_topic))

self.send_signal(self.widget.Inputs.corpus, self.corpus)
self.process_events(until)
self.wait_until_finished()

self.send_signal(self.widget.Inputs.corpus, None)
output = self.get_output(self.widget.Outputs.selected_topic)
self.assertIsNone(output)

def test_saved_selection(self):
def until(widget=self.widget):
return bool(self.get_output(widget.Outputs.selected_topic,
widget=widget))

self.send_signal(self.widget.Inputs.corpus, self.corpus)
self.process_events(until)
self.wait_until_finished()

idx = self.widget.topic_desc.model().index(2, 0)
self.widget.topic_desc.selectionModel().select(
idx, QItemSelectionModel.Rows | QItemSelectionModel.ClearAndSelect)
Expand All @@ -43,7 +33,7 @@ def until(widget=self.widget):

w = self.create_widget(OWTopicModeling, stored_settings=state)
self.send_signal(w.Inputs.corpus, self.corpus, widget=w)
self.process_events(lambda: until(w))

output2 = self.get_output(w.Outputs.selected_topic, widget=w)
# gensim uses quicksort, so sorting is unstable
m1 = output1.metas[output1.metas[:, 0].argsort()]
Expand All @@ -55,17 +45,30 @@ def until(widget=self.widget):

def test_all_topics_output(self):
# LSI produces 9 topics for deerwester, output should be 9
def until(widget=self.widget):
return bool(self.get_output(widget.Outputs.selected_topic,
widget=widget))

self.send_signal(self.widget.Inputs.corpus, self.corpus)
self.process_events(until)
output = self.get_output(self.widget.Outputs.all_topics)

self.assertEqual(len(output), self.widget.model.actual_topics)
self.assertEqual(output.metas.shape[1],
self.widget.corpus.metas.shape[1] + 1)

def test_topic_evaluation(self):
self.send_signal(self.widget.Inputs.corpus, self.corpus)
self.wait_until_finished()

# test LSI
self.assertEqual(self.widget.perplexity, "n/a")
self.assertNotEqual(self.widget.coherence, "n/a")

# test LDA, which is the only one with log perplexity
self.widget.method_index = 1
self.widget.commit()
self.wait_until_finished()

self.assertNotEqual(self.widget.perplexity, "n/a")
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This works in the widget, but not here. I assume I am not calling the method change correctly. Help?

self.assertTrue(self.widget.coherence)


if __name__ == "__main__":
unittest.main()