Merge pull request #815 from VesnaT/keywords_pp_fix

[FIX] Keywords: Always invoke preprocessor __call__
biolab · Apr 21, 2022 · 3a49f97 · 3a49f97
2 parents 88af9f5 + e537d23
commit 3a49f97
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/orangecontrib/text/widgets/owkeywords.py b/orangecontrib/text/widgets/owkeywords.py
@@ -94,7 +94,9 @@ def callback(i: float, status=""):
         # Normalize words
         for preprocessor in corpus.used_preprocessor.preprocessors:
             if isinstance(preprocessor, BaseNormalizer):
-                words = [preprocessor.normalizer(w) for w in words]
+                dummy = Corpus(Domain((), metas=[StringVariable("Words")]),
+                               metas=np.array(words)[:, None])
+                words = list(preprocessor(dummy).tokens.flatten())
 
         # Filter scores using words
         existing_words = [w for w in set(words) if w in scores.index]

diff --git a/orangecontrib/text/widgets/tests/test_owkeywords.py b/orangecontrib/text/widgets/tests/test_owkeywords.py
@@ -85,7 +85,7 @@ def test_run_with_words(self):
         self.assertEqual(len(results.scores), 42)
 
     def test_run_normalize_words(self):
-        normalizer = WordNetLemmatizer()
+        normalizer = LemmagenLemmatizer()
         corpus = normalizer(self.corpus)
 
         words = ["minor", "tree"]