Skip to content

feat: improve walkthrough #31402

feat: improve walkthrough

feat: improve walkthrough #31402

Workflow file for this run

name: E2E test
on:
workflow_dispatch:
inputs:
cases:
description: 'specific cases to be excuted. Sample: ["./aad/a.tests.ts", "./bot/b.tests.ts"]. Set empty to run all cases'
required: false
sample-branch:
description: "default is 'dev', options: 'main'"
required: true
type: string
default: 'dev'
target-testplan-id:
description: "target testplan id: 24569079. If not set, will not archive ado testplan"
required: false
type: string
default: '24569079'
target-testplan-name:
description: "For example: CY230919. Sync test result to this test plan."
required: false
type: string
get-coverage:
description: "check e2e coverage for fx-core"
required: false
type: boolean
default: false
cli-version:
description: "cli version"
required: false
type: string
default: ""
schedule:
- cron: "0 22 * * *"
pull_request:
permissions:
actions: read
jobs:
setup:
if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.get-coverage == 'false' ) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx') }}
runs-on: ubuntu-latest
environment: engineering
permissions:
id-token: write
contents: read
outputs:
cases: ${{ steps.schedule-cases.outputs.cases || steps.dispatch-cases.outputs.cases || steps.pr-cases.outputs.cases }}
env:
AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
M365_ACCOUNT_NAME: ${{ secrets.TEST_M365_NAME }}
M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_M365_PASSWORD }}
M365_TENANT_ID: ${{ secrets.TEST_M365_TENANT_ID }}
CI_ENABLED: "true"
M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME_2 }}
AUTO_TEST_PLAN_ID: ${{ github.event.inputs.target-testplan-id }}
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: azure/login@v1
with:
client-id: ${{secrets.DEVOPS_CLIENT_ID}}
tenant-id: ${{secrets.DEVOPS_TENANT_ID}}
subscription-id: ${{secrets.DEVOPS_SUB_ID}}
- name: setup project
uses: ./.github/actions/setup-project
- name: List cases for schedule
id: schedule-cases
if: ${{ github.event_name == 'schedule' }}
working-directory: packages/tests/src/e2e
run: |
cases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'`
echo "cases=$cases" >> $GITHUB_OUTPUT
- name: List cases for dispatch
id: dispatch-cases
if: ${{ github.event_name == 'workflow_dispatch' }}
working-directory: packages/tests/src/e2e
run: |
inputCases='${{ github.event.inputs.cases }}'
if [ -z "$inputCases" ]; then
allCases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'`
echo "cases=$allCases" >> $GITHUB_OUTPUT
else
echo "cases=$inputCases" >> $GITHUB_OUTPUT
fi
- name: List cases for pull request
id: pr-cases
if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx' }}
working-directory: packages/tests/src/e2e
run: |
cases=`find ./bot ./frontend -path "*.tests.ts" -and \
-not '(' -path '*/*.dotnet.tests.ts' -or -path '*/Blazor*' ')' \
| jq -Rsc '[split("\n") | .[]| select(.!="")]'`
echo "cases=$cases" >> $GITHUB_OUTPUT
- name: E2E Test clean
working-directory: packages/tests
run: |
npm run test:e2e:clean
- name: Archive Test Plan
if: ${{ github.event.inputs.target-testplan-name != '' }}
working-directory: ./packages/tests
run: |
pnpm install
testplanid=`npx ts-node src/scripts/testPlan.ts obtain vscode ${{ github.event.inputs.target-testplan-name }}`
npx ts-node src/scripts/testPlan.ts archive $testplanid
- name: Upload testplan to artifact
if: ${{ github.event.inputs.target-testplan-name != '' }}
uses: actions/upload-artifact@v4
with:
name: testplan
path: |
./packages/tests/testplan.json
execute-case:
if: ${{ needs.setup.outputs.cases }}
env:
AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
M365_ACCOUNT_NAME: ${{ secrets.TEST_M365_NAME }}
M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_M365_PASSWORD }}
M365_TENANT_ID: ${{ secrets.TEST_M365_TENANT_ID }}
CI_ENABLED: "true"
M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME_2 }}
TEAMSFX_DEBUG_TEMPLATE: "true"
NODE_ENV: "development"
TEAMSFX_AAD_DEPLOY_ONLY: "true"
SIDELOADING_SERVICE_ENDPOINT: ${{ secrets.SIDELOADING_SERVICE_ENDPOINT }}
SIDELOADING_SERVICE_SCOPE: ${{ secrets.SIDELOADING_SERVICE_SCOPE }}
needs: setup
runs-on: ubuntu-latest
environment: engineering
permissions:
id-token: write
contents: read
strategy:
fail-fast: false
matrix:
cases: ${{ fromJson(needs.setup.outputs.cases) }}
name: ${{ matrix.cases }}
outputs:
cases: ${{ matrix.cases }}
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: azure/login@v1
with:
client-id: ${{secrets.DEVOPS_CLIENT_ID}}
tenant-id: ${{secrets.DEVOPS_TENANT_ID}}
subscription-id: ${{secrets.DEVOPS_SUB_ID}}
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 18
- name: Setup node for SPFx project
if: contains(matrix.cases, 'SPFx')
uses: actions/setup-node@v3
with:
node-version: 16
- name: Setup Azure Functions Core Tools For Linux
run: |
curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
sudo mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg
sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-$(lsb_release -cs)-prod $(lsb_release -cs) main" > /etc/apt/sources.list.d/dotnetdev.list'
sudo apt-get update
sudo apt-get install azure-functions-core-tools-4
which func
func --version
- name: Setup .net
uses: actions/setup-dotnet@v3
with:
dotnet-version: |
6.0.x
8.0.x
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- uses: pnpm/action-setup@v4
- name: Setup project
run: |
npm run setup
- name: install cli with specific version
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.cli-version != '' }}
run: |
npm install -g @microsoft/teamsapp-cli@${{ github.event.inputs.cli-version }}
- name: install cli for schedule
if: ${{ github.event_name == 'schedule' || ( github.event.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && github.event.inputs.cli-version == '' ) }}
run: |
npm install -g @microsoft/teamsapp-cli@alpha
- name: link cli for workflow_dispatch
if: ${{ github.event_name == 'workflow_dispatch' && github.event.ref != 'refs/heads/dev' && github.event.inputs.cli-version == '' }}
run: |
pnpm link --global
- name: Update CLI and legacy-peer-deps for PR cases
if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx' }}
run: |
npm install -g @microsoft/teamsapp-cli@alpha
npm config set legacy-peer-deps false
- name: print system info
run: |
lscpu
- name: Download samples(daily)
if: github.event_name == 'schedule' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false
uses: actions/checkout@v3
with:
repository: OfficeDev/TeamsFx-Samples
ref: dev
path: packages/tests/src/e2e/resource
- name: Download samples(rc)
if: github.event_name == 'workflow_dispatch' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false && contains(matrix.cases, 'SignatureOutlook') == false && contains(matrix.cases, 'ChefBot') == false
uses: actions/checkout@v3
with:
repository: OfficeDev/TeamsFx-Samples
ref: ${{ github.event.inputs.sample-branch }}
path: packages/tests/src/e2e/resource
- name: Download samples from another repo
if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage')
uses: actions/checkout@v3
with:
repository: OfficeDev/Microsoft-Teams-Samples
ref: main
path: packages/tests/src/e2e/resource
- name: Download samples from office repo
if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'SignatureOutlook')
uses: actions/checkout@v3
with:
repository: OfficeDev/Office-Add-in-samples
ref: main
path: packages/tests/src/e2e/resource
- name: Download samples from ai repo
if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ChefBot')
uses: actions/checkout@v3
with:
repository: microsoft/teams-ai
ref: main
path: packages/tests/src/e2e/resource
- name: run test
working-directory: packages/tests/src/e2e
run: |
file=`find . -wholename "${{ matrix.cases }}"`
if [ -z "$file" ]; then
echo "can't find target case in $file"
exit 1
else
npx mocha --reporter mochawesome --timeout 1200000 $file
fi
- name: get report name
id: get-report-name
if: ${{ always() }}
run: |
name="${{ matrix.cases }}"
name="${name//'.tests.ts'/}"
name="${name//.\//}"
name="${name//\//_}"
echo "name=$name" >> $GITHUB_OUTPUT
- name: Upload test report
uses: actions/upload-artifact@v4
if: ${{ github.event_name != 'schedule' || success() || (failure() && github.run_attempt >= 5) }}
with:
name: test-result-${{ steps.get-report-name.outputs.name }}
path: |
./packages/tests/src/e2e/mochawesome-report/mochawesome.json
- name: Download TestPlan
if: ${{ always() && github.event.inputs.target-testplan-name != '' }}
uses: actions/download-artifact@v4
with:
name: testplan
path: ./packages/tests
- name: Sync to Azure DevOps Test Plan
if: ${{ always() && github.event.inputs.target-testplan-name != '' }}
working-directory: packages/tests
run: |
npx ts-node src/scripts/testPlan.ts report ./testplan.json src/e2e/mochawesome-report
tear-down:
needs: [execute-case, e2e-coverage]
if: ${{always() && (toJSON(needs.execute-case.outputs.cases) != 'null' || needs.e2e-coverage.result != 'skipped' )}}
runs-on: ubuntu-latest
env:
AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
M365_ACCOUNT_NAME: ${{ secrets.TEST_M365_NAME }}
M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_M365_PASSWORD }}
M365_TENANT_ID: ${{ secrets.TEST_M365_TENANT_ID }}
CI_ENABLED: "true"
M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME_2 }}
steps:
- name: Checkout (dev)
uses: actions/checkout@v3
- name: setup project
uses: ./.github/actions/setup-project
- name: E2E Test clean
working-directory: packages/tests
run: |
npm run test:e2e:clean
rerun:
permissions:
actions: write
needs: tear-down
if: ${{ (github.event_name == 'schedule' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx')) && failure() && github.run_attempt < 5 }}
runs-on: ubuntu-latest
steps:
- name: rerun
run: |
curl \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ github.repository }}/actions/workflows/rerun.yml/dispatches \
-d '{"ref":"${{ github.ref_name }}","inputs":{"run_id":"${{ github.run_id }}", "max_attempts":"5"}}'
report:
needs: execute-case
if: ${{ github.event_name == 'schedule' && (success() || (failure() && github.run_attempt >= 5)) }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install Dateutils
run: |
sudo apt install dateutils
- uses: actions/download-artifact@v4
with:
path: ~/artifacts
- name: List jobs
id: list-jobs
working-directory: packages/tests
run: |
page=1
jobs="[]"
while :
do
url=https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs\?per_page\=100\&page\=$page
resp=`curl -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url`
new_jobs=`echo $resp | jq -cr '.jobs'`
jobs=`jq -cr --slurp 'add' <(echo "$jobs") <(echo "$new_jobs")`
has_next=`curl -I -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url | grep -Fi "link:" | grep "rel=\"last\"" || true`
if [ -z "$has_next" ]; then
break
fi
page=$((page+1))
done
cases=`echo $jobs| jq -r '.[] | select(.name | contains("tests.ts")) | .name'`
passed=0
failed=0
skipped=0
passed_lists=""
failed_lists=""
skipped_lists=""
emails="[email protected];"
while IFS= read -r case;
do
if [ -z "$case" ]; then
continue
fi
file="src/e2e/$case"
elegant_path="${file//.\//}"
started_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .started_at'`
completed_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .completed_at'`
duration=`dateutils.ddiff $started_at $completed_at -f "%Mm %Ss"`
email=""
if grep -q "@author" $file; then
email=`grep '@author' $file | grep -i -o '[A-Z0-9._%+-]\+@[A-Z0-9.-]\+\.[A-Z]\{2,4\}'`
fi
author=""
if [ -z "$email" ]; then
author="N/A"
else
author="<a href=\\\"mailto:$email\\\"><span>$email</span></a>"
fi
url=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .html_url'`
url="<a href=\\\"$url\\\">$elegant_path</a>"
target_type="TS/JS"
if [[ $case == *".dotnet."* ]]; then
target_type=".NET"
fi
name="${case//'.tests.ts'/}"
name="${name//.\//}"
name="${name//\//_}"
job_id=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .id'`
report_file=`find ~/artifacts -wholename "*${name}*/mochawesome.json"`
if [ ! -z "$report_file" ]; then
echo "Found the $report_file with $job_id and $name"
tests=`cat $report_file | jq -cr '[ .. | objects | with_entries(select(.key=="tests")) | select(. != {}) | select(.tests | type=="array") ] | map(.tests | .[]) | .[]'`
lable=""
while IFS= read -r test;
do
name=`echo $test | jq -cr .fullTitle`
duration=`echo $test | jq -cr .duration`
if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then
passed=$((passed+1))
label="<span style=\\\"background-color:#2aa198;color:white;font-weight:bold;\\\">PASSED</span>"
elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then
failed=$((failed+1))
label="<span style=\\\"background-color: #dc322f;color:white;font-weight:bold;\\\">FAILED</span>"
if [[ ! -z "$email" && ! "$emails" == *"$email"* ]]; then
emails="$emails;$email;[email protected];[email protected]"
fi
elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then
skipped=$((skipped+1))
label="<span style=\\\"background-color: #b58900;color:white;font-weight:bold;\\\">SKIPPED</span>"
fi
row="<tr> <td style=\\\"text-align: left;\\\">$url</td> <td style=\\\"text-align: left;\\\">$name</td> <td style=\\\"text-align: center;\\\">$target_type</td> <td style=\\\"text-align: center;\\\">$label</td> <td style=\\\"text-align: center;\\\">$author</td> <td>$((duration/1000)) sec</td> </tr>"
if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then
passed_lists="$passed_lists $row"
elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then
failed_lists="$failed_lists $row"
elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then
skipped_lists="$skipped_lists $row"
fi
done <<< $tests
else
echo "Failed to find the $report_file with $job_id and $name"
fi
done <<< $cases
body="Dashboard App: <a href=\\\"https:\/\/teams.microsoft.com\/l\/entity\/c439ae8d-3ab3-4efd-9223-87366d8c170c\/_djb2_msteams_prefix_1252604900?context=%7B%22channelId%22%3A%2219%3A79488ced607f4fbf8d8433e931cad176%40thread.tacv2%22%7D&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47\\\">Click Here to Open Dashboard App</a> <table class=\\\"w3-table w3-striped w3-bordered\\\"> <tr> <th>PATH</th> <th>CASE</th> <th>TARGET TYPE</th> <th>STATUS</th> <th>AUTHOR</th> <th>DURATION</th> </tr> $failed_lists $skipped_lists $passed_lists </table> <br />"
total=$((passed+failed+skipped))
subject="TeamsFx E2E V3 Test Report ($passed/$total Passed, $failed/$total Failed, $skipped/$total Skipped)"
if [ $failed -gt 0 ]; then
subject="[FAILED] $subject"
else
subject="[PASSED] $subject"
fi
echo "body=$body" >> $GITHUB_OUTPUT
echo "to=$emails" >> $GITHUB_OUTPUT
echo "subject=$subject" >> $GITHUB_OUTPUT
- name: Send E-mail to the whole team
uses: ./.github/actions/send-email-report
env:
TO: ${{ steps.list-jobs.outputs.to }}
BODY: '"${{ steps.list-jobs.outputs.body }}"'
SUBJECT: ${{ steps.list-jobs.outputs.subject }}
MAIL_CLIENT_ID: ${{ secrets.TEST_CLEAN_CLIENT_ID }}
MAIL_CLIENT_SECRET: ${{ secrets.TEST_CLEAN_CLIENT_SECRET }}
MAIL_TENANT_ID: ${{ secrets.TEST_CLEAN_TENANT_ID }}
e2e-coverage:
if: ${{github.event_name == 'workflow_dispatch' && github.event.inputs.get-coverage == 'true' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 18
- uses: pnpm/action-setup@v4
- name: mv files
working-directory: packages/tests
run: |
rm src/e2e/commonUtils.ts src/utils/commonUtils.ts
mv src/e2e/commonUtils.ts.back src/e2e/commonUtils.ts
mv src/utils/commonUtils.ts.back src/utils/commonUtils.ts
- uses: jossef/action-set-json-field@v1
with:
file: ./packages/fx-core/.nycrc
field: excludeAfterRemap
value: false
- name: Update config.json
working-directory: packages/fx-core
run: echo "`jq '.include=["build"]' .nycrc`" > .nycrc
- name: less nycrc
working-directory: packages/fx-core
run: |
less .nycrc
- name: setup
run: |
npm run setup
- name: Download samples(daily)
working-directory: packages/tests/src/e2e
run: |
mkdir resource && cd resource
wget https://github.com/OfficeDev/TeamsFx-Samples/archive/refs/heads/dev.zip
unzip dev.zip -d .
mv TeamsFx-Samples-dev/* .
rm -rf dev.zip TeamsFx-Samples-dev
- name: run test for coverage
working-directory: packages/fx-core
env:
AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }}
CI_ENABLED: "true"
M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }}
TEAMSFX_DEBUG_TEMPLATE: "true"
NODE_ENV: "development"
TEAMSFX_AAD_DEPLOY_ONLY: "true"
SIDELOADING_SERVICE_ENDPOINT: ${{ secrets.SIDELOADING_SERVICE_ENDPOINT }}
SIDELOADING_SERVICE_SCOPE: ${{ secrets.SIDELOADING_SERVICE_SCOPE }}
run: |
npx nyc mocha --parallel ../tests/src/e2e/*/*.tests.ts
- name: pack zip
if: ${{always()}}
working-directory: packages/fx-core
run: |
zip -r out1.zip .nyc_output/
zip -r out2.zip coverage/
- uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: coverage
path: |
packages/fx-core/out1.zip
packages/fx-core/out2.zip
- name: CodeCov report attempt
if: ${{always()}}
continue-on-error: true
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true
verbose: true