Tested interactively on runner, now trying e2e again #287
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End tests | |
| #on: [push, pull_request] | |
| on: [push] | |
| #on: | |
| # pull_request_target: | |
| # types: [labeled] | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| environment: "GitHub Actions 1" | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ASSISTANTS_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ASSISTANTS_API_TYPE: ${{ secrets.ASSISTANTS_API_TYPE }} | |
| ASSISTANTS_ID: ${{ secrets.ASSISTANTS_ID }} | |
| ASSISTANTS_BASE_URL: ${{ secrets.ASSISTANTS_BASE_URL }} | |
| ASSISTANTS_MODEL: ${{ secrets.ASSISTANTS_MODEL }} | |
| ASSISTANTS_BOT_NAME: ${{ secrets.ASSISTANTS_BOT_NAME }} | |
| POSTGRES_DATA_HOST: ${{ secrets.POSTGRES_DATA_HOST }} | |
| POSTGRES_DATA_PORT: ${{ secrets.POSTGRES_DATA_PORT }} | |
| POSTGRES_DATA_DB: ${{ secrets.POSTGRES_DATA_DB }} | |
| POSTGRES_DATA_USER: ${{ secrets.POSTGRES_DATA_USER }} | |
| POSTGRES_DATA_PASSWORD: ${{ secrets.POSTGRES_DATA_PASSWORD }} | |
| DATA_DB_CONN_STRING: ${{ secrets.DATA_DB_CONN_STRING }} | |
| POSTGRES_RECIPE_HOST: ${{ secrets.POSTGRES_RECIPE_HOST }} | |
| POSTGRES_RECIPE_PORT: ${{ secrets.POSTGRES_RECIPE_PORT }} | |
| POSTGRES_RECIPE_DB: ${{ secrets.POSTGRES_RECIPE_DB }} | |
| POSTGRES_RECIPE_USER: ${{ secrets.POSTGRES_RECIPE_USER }} | |
| POSTGRES_RECIPE_PASSWORD: ${{ secrets.POSTGRES_RECIPE_PASSWORD }} | |
| RECIPE_DB_CONN_STRING: "postgresql://${{ secrets.POSTGRES_RECIPE_USER }}:${{ secrets.POSTGRES_RECIPE_PASSWORD }}@${{ secrets.POSTGRES_RECIPE_HOST }}:${{ secrets.POSTGRES_RECIPE_PORT }}/${{ secrets.POSTGRES_RECIPE_DB }}" | |
| RECIPES_OPENAI_API_TYPE: ${{ secrets.RECIPES_OPENAI_API_TYPE }} | |
| RECIPES_OPENAI_API_KEY: ${{ secrets.RECIPES_OPENAI_API_KEY }} | |
| RECIPES_MODEL: ${{ secrets.RECIPES_MODEL }} | |
| RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME: ${{ secrets.RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME }} | |
| RECIPES_MEMORY_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_MEMORY_SIMILARITY_CUTOFF }} | |
| RECIPES_RECIPE_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_RECIPE_SIMILARITY_CUTOFF }} | |
| RECIPES_HELPER_FUNCTION_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_HELPER_FUNCTION_SIMILARITY_CUTOFF }} | |
| RECIPES_MODEL_TEMP: ${{ secrets.RECIPES_MODEL_TEMP }} | |
| RECIPES_MODEL_MAX_TOKENS: ${{ secrets.RECIPES_MODEL_MAX_TOKENS }} | |
| IMAGE_HOST: ${{ secrets.IMAGE_HOST }} | |
| RECIPE_SERVER_API: ${{ secrets.RECIPE_SERVER_API }} | |
| CHAT_URL: ${{ secrets.CHAT_URL }} | |
| CHAINLIT_AUTH_SECRET: ${{ secrets.CHAINLIT_AUTH_SECRET }} | |
| USER_LOGIN: ${{ secrets.USER_LOGIN }} | |
| USER_PASSWORD: ${{ secrets.USER_PASSWORD }} | |
| COLUMNS: 150 | |
| CMD: "docker-compose -f ./docker-compose-github.yml exec promptflow" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - name: Spin up Docker containers | |
| run: | | |
| env > .env | |
| echo "Installing demo data ..." | |
| # Get demo data | |
| pip3 install gdown==5.2.0 | |
| cd data && python3 download_demo_data.py && cd .. | |
| # TODO this should be enhanced to use a buildx bake to leverage layer caching for faster builds, or push to repo and simply have a pull for the run | |
| # TODO docker-compose files should be refactored to use scopes instead of different versions for each environment | |
| echo "Starting docker containers for dbs and server ..." | |
| docker-compose -f ./docker-compose-github.yml pull | |
| docker-compose -f ./docker-compose-github.yml up -d --build | |
| # TODO: For some reason, maybe buildkit, in Github docker compose builds the image differently, and it doesn't work. Individual image build works. | |
| docker build --build-arg OPENAI_API_KEY=$OPENAI_API_KEY --build-arg CHAT_URL=$CHAT_URL --build-arg OPENAI_API_ENDPOINT=$OPENAI_API_ENDPOINT --no-cache -t promptflow -f ./flows/chainlit-ui-evaluation/Dockerfile . | |
| docker run --env USER_LOGIN=${USER_LOGIN} --env USER_PASSWORD=${USER_PASSWORD} --env CHAT_URL=${CHAT_URL} --network=data-recipes-ai_default -d --name promptflow promptflow | |
| docker ps | |
| echo "logs datadb ..." | |
| docker-compose -f docker-compose-github.yml logs datadb | |
| echo "logs promptflow ..." | |
| docker logs promptflow | |
| docker ps | |
| # Get ssh connection details for runner | |
| # See here https://github.com/marketplace/actions/debugging-with-ssh | |
| # Basically, uncomment this, then get connection string in actions output, then connect with | |
| #ssh -i <YOUR GITHUB SSH KEY> <CONN STRING ON ACTIONS> | |
| #- name: Setup upterm session | |
| # uses: lhotari/action-upterm@v1 | |
| - name: Run tests | |
| run: | | |
| env > .env | |
| #docker-compose -f ./docker-compose-github.yml exec -T promptflow pf run create --flow . --data ./data.jsonl --stream --column-mapping query='${data.query}' context='${data.context}' chat_history='${data.chat_history}' --name base_run | |
| docker-compose exec -T promptflow pf run create --flow . --data ./data.jsonl --stream --column-mapping query='${data.query}' context='${data.context}' chat_history='${data.chat_history}' --name base_run | |
| - name: Show results | |
| run: | | |
| docker-compose exec -T promptflow pf run show-details -n base_run | |
| echo "Getting metrics ..." | |
| docker-compose exec -T promptflow pf run show-metrics -n base_run | |
| ##docker-compose exec -T promptflow pf run visualize -n base_run | |
| echo "Checking results ..." | |
| docker-compose exec -T promptflow python3 check_evaluation_results.py | |