Add the ability to score the tokenizer training set as a test set - m… #6331

	name: Run Stanza Tests
	on: [push]
	jobs:
	Run-Stanza-Tests:
	runs-on: self-hosted
	steps:
	- run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
	- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
	- run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
	- name: Check out repository code
	uses: actions/checkout@v2
	- run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
	- run: echo "🖥️ The workflow is now ready to test your code on the runner."
	- name: Run Stanza Tests
	run: \|
	# set up environment
	echo "Setting up environment..."
	bash
	#. $CONDA_PREFIX/etc/profile.d/conda.sh
	. /home/stanzabuild/miniconda3/etc/profile.d/conda.sh
	conda activate stanza
	export STANZA_TEST_HOME=/scr/stanza_test
	export CORENLP_HOME=$STANZA_TEST_HOME/corenlp_dir
	export CLASSPATH=$CORENLP_HOME/*:
	echo CORENLP_HOME=$CORENLP_HOME
	echo CLASSPATH=$CLASSPATH
	# install from stanza repo being evaluated
	echo PWD: $pwd
	echo PATH: $PATH
	pip3 install -e .
	pip3 install -e .[test]
	pip3 install -e .[transformers]
	pip3 install -e .[tokenizers]
	# set up for tests
	echo "Running stanza test set up..."
	rm -rf $STANZA_TEST_HOME
	python3 stanza/tests/setup.py
	# run tests
	echo "Running tests..."
	export CUDA_VISIBLE_DEVICES=2
	pytest stanza/tests

	- run: echo "🍏 This job's status is ${{ job.status }}."

Provide feedback