preprocess: add udpipe tokenizer to report

biolab · Sep 4, 2018 · d64ed16 · d64ed16
1 parent a4e2daf
commit d64ed16
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/orangecontrib/text/preprocess/preprocess.py b/orangecontrib/text/preprocess/preprocess.py
@@ -134,7 +134,10 @@ def __str__(self):
         return '\n'.join(['{}: {}'.format(name, value) for name, value in self.report()])
 
     def report(self):
-        return (
+        if getattr(self.normalizer, 'use_tokenizer', False):
+            self.tokenizer = \
+                'UDPipe Tokenizer ({})'.format(self.normalizer.language)
+        rep = (
             ('Transformers', ', '.join(str(tr) for tr in self.transformers)
             if self.transformers else None),
             ('Tokenizer', str(self.tokenizer) if self.tokenizer else None),
@@ -147,6 +150,8 @@ def report(self):
             else None),
             ('Pos tagger', str(self.pos_tagger) if self.pos_tagger else None),
         )
+        del self.tokenizer
+        return rep
 
 
 base_preprocessor = Preprocessor(transformers=BASE_TRANSFORMERS,