preprocess: add udpipe tokenizer to report

biolab · Sep 3, 2018 · ab07afe · ab07afe
1 parent 86b99d3
commit ab07afe
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/orangecontrib/text/preprocess/preprocess.py b/orangecontrib/text/preprocess/preprocess.py
@@ -131,7 +131,10 @@ def __str__(self):
         return '\n'.join(['{}: {}'.format(name, value) for name, value in self.report()])
 
     def report(self):
-        return (
+        if getattr(self.normalizer, 'use_tokenizer', False):
+            self.tokenizer = \
+                'UDPipe Tokenizer ({})'.format(self.normalizer.language)
+        rep = (
             ('Transformers', ', '.join(str(tr) for tr in self.transformers)
             if self.transformers else None),
             ('Tokenizer', str(self.tokenizer) if self.tokenizer else None),
@@ -144,7 +147,8 @@ def report(self):
             else None),
             ('Pos tagger', str(self.pos_tagger) if self.pos_tagger else None),
         )
-
+        del self.tokenizer
+        return rep
 
 base_preprocessor = Preprocessor(transformers=BASE_TRANSFORMERS,
                                  tokenizer=BASE_TOKENIZER)