Add new or updated feeds from Google Sheets/Form #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Add new or updated feeds from Google Sheets/Form | |
| on: | |
| schedule: | |
| - cron: '0 4 * * *' # At 00:00 ETC every day | |
| env: | |
| DATE_FORMAT: "[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}" # this is the format we need to compare dates between the CSV and the local system. | |
| DATE_FORMAT_DESIRED: "MM/dd/yyyy" | |
| USERNAME: "github-actions[bot]" # GitHub username that will create the PR | |
| USERNAME_EMAIL: "41898282+github-actions[bot]@users.noreply.github.com" | |
| ORGANIZATION: MobilityData # organization name | |
| REPO_NAME: mobity-database-catalogs # repository name | |
| BASE: "main" | |
| REVIEWERS_JSON: "[\"emmambd\"]" # List of GitHub usernames of the reviewers, in a JSON array : ["username1", "username2"] | |
| GTFS_SCHEDULE_CATALOG_PATH_FROM_ROOT: "catalogs/sources/gtfs/schedule/" | |
| GTFS_REALTIME_CATALOG_PATH_FROM_ROOT: "catalogs/sources/gtfs/realtime/" | |
| jobs: | |
| add-new-updated-feeds: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup global variables | |
| id: global_vars | |
| run: | | |
| echo "TODAYS_DATE=$(date +%m/%d/%Y)" >> $GITHUB_ENV # Ex.: 07/27/2023 | |
| echo "TODAYS_DAY=$(date '+%d')" >> $GITHUB_ENV # Ex.: 27 | |
| echo "TODAYS_MONTH=$(date '+%m')" >> $GITHUB_ENV # Ex.: 07 | |
| echo "TODAYS_YEAR=$(date '+%Y')" >> $GITHUB_ENV # Ex.: 2023 | |
| - name: Create branch name | |
| id: create_branch_name | |
| run: | | |
| echo "BRANCH=${{ env.TODAYS_YEAR }}-${{ env.TODAYS_MONTH }}-${{ env.TODAYS_DAY }}" >> $GITHUB_OUTPUT # Branch name | |
| - name: Load secrets from 1Password | |
| id: onepw_secrets | |
| uses: 1password/[email protected] | |
| with: | |
| export-env: true # Export loaded secrets as environment variables | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential" | |
| # - name: Get swift version # just for verification, can leave commented out | |
| # run: swift --version | |
| # Swift is installed by default on github runners, tested with v 5.8.1 | |
| - name: Checkout repo | |
| id: checkout_repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.BASE }} | |
| fetch-depth: 0 | |
| token: ${{ env.CREDENTIALS }} | |
| - name: Create new branch | |
| shell: bash | |
| run: | | |
| git checkout -b ${{ steps.create_branch_name.outputs.BRANCH }} | |
| git reset --hard ${{ env.BASE }} | |
| - name: Download CSV and process each lines | |
| id: process-csv | |
| run: | | |
| cd ${{ github.workspace }}/scripts | |
| OUTPUT=$(swift process_csv_in_github_action.swift "${{ secrets.CSV_URL }}" "${{ env.DATE_FORMAT }}" "${{ env.DATE_FORMAT_DESIRED }}") | |
| echo "PYTHON_SCRIPT_ARGS=${OUTPUT}" >> $GITHUB_OUTPUT | |
| - name: Setup Python | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.11' # install the python version needed | |
| - name: Create + activate a Python virtual env & run script | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/tools | |
| PYTHONIOENCODING: "utf8" #ascii | |
| shell: bash | |
| run: | | |
| python -m venv env | |
| source env/bin/activate | |
| pip install virtualenv --quiet | |
| pip install gtfs_kit --quiet | |
| pip install unidecode --quiet | |
| # We use § as the separator because for an unknown reason a newline doesn't work. | |
| sections=$(echo '${{ steps.process-csv.outputs.PYTHON_SCRIPT_ARGS }}' | awk -F'§' '{for (i=1; i<=NF; i++) print $i}') | |
| for section in "${sections[@]}"; do | |
| eval "python -c 'from tools.operations import *; ${section}'" | |
| done | |
| - name: Commit & push | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| uses: EndBug/[email protected] | |
| with: | |
| github_token: ${{ env.CREDENTIALS }} | |
| new_branch: ${{ steps.create_branch_name.outputs.BRANCH }} | |
| author_name: ${{ env.USERNAME }} | |
| author_email: ${{ env.USERNAME_EMAIL }} | |
| committer_name: ${{ env.USERNAME }} | |
| committer_email: ${{ env.USERNAME_EMAIL }} | |
| message: "Automated commit — New/Updated feed" | |
| # - name: Create Pull Request | |
| # if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| # uses: peter-evans/[email protected] | |
| # with: | |
| # token: ${{ env.CREDENTIALS }} | |
| # title: "Automated commit — New/Updated feed" | |
| # commit-message: "Automated commit — New/Updated feed" | |
| # body: "New feed(s) were found, and added as a PR for you to review." | |
| # author: "${{ env.USERNAME }} <${{ env.USERNAME_EMAIL }}>" | |
| # reviewers: ${{ env.REVIEWERS_JSON }} | |
| # branch: ${{ steps.create_branch_name.outputs.BRANCH }} | |
| # base: ${{ env.BASE }} | |
| # add-paths: | | |
| # ${{ env.GTFS_SCHEDULE_CATALOG_PATH_FROM_ROOT }} | |
| # ${{ env.GTFS_REALTIME_CATALOG_PATH_FROM_ROOT }} |