Merge pull request #1483 from PaddlePaddle/update-faster-tokenizer-map

Support ppminilm in FasterTokenizer
PaddlePaddle · Dec 17, 2021 · f56e429 · f56e429
2 parents 374d31b + 10fc6a6
commit f56e429
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/paddlenlp/experimental/faster_tokenizer.py b/paddlenlp/experimental/faster_tokenizer.py
@@ -71,6 +71,7 @@ class FasterTokenizer(nn.Layer):
         "ernie-1.0": ErnieTokenizer,
         "ernie-2.0-en": ErnieTokenizer,
         "ernie-2.0-large-en": ErnieTokenizer,
+        "ppminilm-6l-768h": ErnieTokenizer,
         "roberta-wwm-ext": RobertaTokenizer,
         "roberta-wwm-ext-large": RobertaTokenizer,
         "rbt3": RobertaTokenizer,