Skip to content

Commit

Permalink
Test that the contextual lemmatizer is correctly loaded and functions…
Browse files Browse the repository at this point in the history
… without crashing
  • Loading branch information
AngledLuffa committed Dec 24, 2024
1 parent af3d42b commit 8fd08b3
Showing 1 changed file with 10 additions and 0 deletions.
10 changes: 10 additions & 0 deletions stanza/tests/pipeline/test_lemmatizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,13 @@ def test_latin_caseless_lemmatizer():
assert len(doc.sentences[0].words) == 3
for word, expected in zip(doc.sentences[0].words, expected_lemmas):
assert word.lemma == expected

def test_contextual_lemmatizer():
nlp = stanza.Pipeline('en', processors='tokenize,pos,lemma', model_dir=TEST_MODELS_DIR, package={"lemma": "default_accurate"}, download_method="reuse_resources")
lemmatizer = nlp.processors['lemma']._trainer
# the accurate model should have a 's classifier
assert len(lemmatizer.contextual_lemmatizers) > 0
# ideally the doc would have 'have' as the lemma for the second
# word, but maybe it's not always accurate. actually, it works
# fine at the time of this test
doc = nlp("He's added a contextual lemmatizer")

0 comments on commit 8fd08b3

Please sign in to comment.