Skip to content

Sync dry-run from dist-$STACK-develop/ to dist-$STACK-stable/ #103

Sync dry-run from dist-$STACK-develop/ to dist-$STACK-stable/

Sync dry-run from dist-$STACK-develop/ to dist-$STACK-stable/ #103

Workflow file for this run

name: Platform packages sync from -develop/ to -stable/
run-name: Sync${{ inputs.dry-run == true && ' dry-run' || '' }} from dist-$STACK-develop/ to dist-$STACK-stable/
env:
stacks_list_for_shell_expansion: "{heroku-20,heroku-22,heroku-24-amd64,heroku-24-arm64}"
src_path_suffix: "-develop/"
dst_path_suffix: "-stable/"
on:
workflow_dispatch:
inputs:
stack-heroku-20:
description: 'Sync heroku-20 packages'
type: boolean
default: true
required: false
stack-heroku-22:
description: 'Sync heroku-22 packages'
type: boolean
default: true
required: false
stack-heroku-24-amd64:
description: 'Sync heroku-24 (amd64) packages'
type: boolean
default: true
required: false
stack-heroku-24-arm64:
description: 'Sync heroku-24 (arm64) packages'
type: boolean
default: true
required: false
dry-run:
description: 'Only list package changes, without syncing'
type: boolean
default: false
required: false
update-devcenter:
description: 'Update "PHP Support" Dev Center article after sync'
type: boolean
default: true
required: false
permissions:
contents: read
jobs:
stack-list:
runs-on: ubuntu-24.04
outputs:
stacks: ${{ steps.list-stacks.outputs.matrix }}
steps:
- id: list-stacks
name: Generate list of stacks to sync based on input checkboxes
run: |
echo '## Stacks to sync' >> "$GITHUB_STEP_SUMMARY"
set -o pipefail
stacks=(${{ inputs.stack-heroku-20 == true && 'heroku-20' || ''}} ${{ inputs.stack-heroku-22 == true && 'heroku-22' || ''}} ${{ inputs.stack-heroku-24-amd64 == true && 'heroku-24-amd64' || ''}} ${{ inputs.stack-heroku-24-arm64 == true && 'heroku-24-arm64' || ''}})
printf -- "- %s\n" "${stacks[@]}" >> "$GITHUB_STEP_SUMMARY"
echo -n "matrix=" >> "$GITHUB_OUTPUT"
printf "%s\n" "${stacks[@]}" | jq -jcRn '[inputs|select(length>0)]' >> "$GITHUB_OUTPUT"
docker-build:
needs: stack-list
if: ${{ needs.stack-list.outputs.stacks != '[]' && needs.stack-list.outputs.stacks != '' }}
runs-on: ${{ endsWith(matrix.stack, '-arm64') && 'pub-hk-ubuntu-24.04-arm-small' || 'ubuntu-24.04' }}
strategy:
matrix:
stack: ${{ fromJSON(needs.stack-list.outputs.stacks) }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Cache Docker build
id: cache-docker
uses: actions/cache@v4
with:
key: docker-cache-heroku-php-build-${{matrix.stack}}.${{github.sha}}
path: /tmp/docker-cache.tar.gz
- name: Build Docker image
if: steps.cache-docker.outputs.cache-hit != 'true'
# our "input" stack might contain a "-amd64" or "-arm64" suffix, which we strip off for the Dockerfile name
run: |
shopt -s extglob
stackname_with_architecture=${{matrix.stack}}
docker build --tag heroku-php-build-${stackname_with_architecture}:${{github.sha}} --file support/build/_docker/${stackname_with_architecture%-?(amd|arm)64}.Dockerfile .
- name: Save built Docker image
if: steps.cache-docker.outputs.cache-hit != 'true'
run: docker save heroku-php-build-${{matrix.stack}}:${{github.sha}} | gzip -1 > /tmp/docker-cache.tar.gz
sync:
needs: [stack-list, docker-build]
strategy:
matrix:
stack: ${{ fromJSON(needs.stack-list.outputs.stacks) }}
runs-on: ${{ endsWith(matrix.stack, '-arm64') && 'pub-hk-ubuntu-24.04-arm-small' || 'ubuntu-24.04' }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Restore cached Docker build
uses: actions/cache/restore@v4
with:
key: docker-cache-heroku-php-build-${{matrix.stack}}.${{github.sha}}
path: /tmp/docker-cache.tar.gz
- name: Load cached Docker image
run: docker load -i /tmp/docker-cache.tar.gz
- name: ${{ inputs.dry-run == true && 'Dry-run sync of' || 'Sync' }} changed packages to production bucket
run: |
# we want to fail if 'docker run' fails; without this, 'tee' would "eat" the failure status
set -o pipefail
# yes gets "n" to print for dry-runs so the sync aborts
# errors are redirected to /dev/null, and we || true, to suppress SIGPIPE errors from 'docker run' exiting eventually
# we need -i for Docker to accept input on stdin, but must not use -t for the pipeline to work
(yes "${{ inputs.dry-run == true && 'n' || 'y' }}" 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{matrix.stack}}:${{github.sha}} sync.sh lang-php dist-${{matrix.stack}}${{env.dst_path_suffix}} 2>&1 | tee sync-${{matrix.stack}}.log
- name: Upload sync log as artifact
uses: actions/upload-artifact@v4
with:
name: synclog-${{matrix.stack}}
path: sync-${{matrix.stack}}.log
- name: Output dry-run summary
if: ${{ inputs.dry-run == true }}
run: |
echo '## Package changes available for syncing to ${{matrix.stack}} production bucket' >> "$GITHUB_STEP_SUMMARY"
echo '> [!IMPORTANT]' >> "$GITHUB_STEP_SUMMARY"
echo '> **This is output from a dry-run**, no changes have been synced to production!' >> "$GITHUB_STEP_SUMMARY"
echo >> "$GITHUB_STEP_SUMMARY"
echo '```' >> "$GITHUB_STEP_SUMMARY"
sed -n '/^The following packages will/,/POTENTIALLY DESTRUCTIVE ACTION/{/POTENTIALLY DESTRUCTIVE ACTION/!p}' sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY"
echo '```' >> "$GITHUB_STEP_SUMMARY"
- name: Output sync summary
if: ${{ inputs.dry-run == false }}
run: |
echo '## Package changes synced to ${{matrix.stack}} production bucket' >> "$GITHUB_STEP_SUMMARY"
echo '```' >> "$GITHUB_STEP_SUMMARY"
cat sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY"
echo '```' >> "$GITHUB_STEP_SUMMARY"
devcenter-generate:
needs: sync
runs-on: ubuntu-24.04
outputs:
diff_result: ${{ steps.diff.outputs.diff_result }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dos2unix
run: |
sudo apt-get update
sudo apt-get install dos2unix
- name: Install PHP and Composer
uses: shivammathur/setup-php@v2
with:
php-version: "8.2"
tools: "composer:2.8"
- name: Install Dev Center generator dependencies
run: |
composer install -d support/devcenter/
- name: Generate Dev Center article sections
run: |
set -o pipefail
urls=( https://lang-php.s3.amazonaws.com/dist-${{ env.stacks_list_for_shell_expansion }}${{ inputs.dry-run == true && env.src_path_suffix || env.dst_path_suffix }}packages.json )
# generate.php can generate individual sections, but doing it in one go a) is faster and b) means this code does not need to know what those sections are
# Instead we split the generated contents into individual files, with the known delimiter as the split pattern.
# tee stderr to a file
support/devcenter/generate.php "${urls[@]}" 2> >(tee warnings.txt >&2) | csplit -s -z -f 'section-' -b '%02d.md' - '/^<!-- BEGIN [A-Z_][A-Z0-9_-]\+ -->$/' '{*}'
# if there were warnings or notices on stderr, output them for GitHub highlighting
# the \L, \E and \u extensions are case modifiers
# this turns "(WARNING|NOTICE): foobar" into "::(warning|notice) title=(Warning|Notice) emitted by generator.php::foobar"
sed -E -e 's/^(WARNING|NOTICE): /::\L\1\E title=\L\u\1\E emitted by generator.php::/g' warnings.txt
# sanity check number of generated splits (e.g. in case the split ever changes)
shopt -s nullglob
splits=( section-*.md )
if (( ${#splits[@]} < 2 )); then
echo 'error::Expected more than one section from generator.'
exit 1
fi
- name: Download current Dev Center article markdown
run: |
set -o pipefail
# jq -j, not -r, otherwise we get a stray trailing newline
curl -H "Accept: application/json" https://devcenter.heroku.com/api/v1/articles/php-support | jq -j '.content' > php-support.md
# Because the articles are edited in a web interface, they likely use CRLF line endings.
# We will be patching using the LF line ending section files generated in an earlier step.
# For this reason, we may have to convert to LF, so we check if the file would be converted by dos2unix using the --info option.
# The "c" info flag prints only file names that would trigger conversion; we first remember this output for the next step via tee.
# The "0" flag triggers zero-byte output for happy consumption by xargs
# Then, we finally run the conversion (if needed) by passing the file name to dos2unix again via xargs.
dos2unix --info=c0 php-support.md | tee have_crlf.txt | xargs -r0 dos2unix
- name: Find generated section start/end markers in Dev Center article markdown
id: find-section-markers
run: |
# init job file
echo -n > php-support.md.ed-unordered.txt
for f in section-*.md; do
# extract first and last lines of the section file (those are the start and end markers)
first=$(head -n1 "$f")
last=$(tail -n1 "$f")
# grep the line numbers (-n) as fixed (-F) full-line (-x) strings and extract them
start=$(set -o pipefail; grep -nFx "$first" php-support.md | cut -d':' -f1) || {
echo "::warning title=Failed to match section start marker::Start marker '$first' not found in input markdown; skipping '$f'..."
continue
}
end=$(set -o pipefail; grep -nFx "$last" php-support.md | cut -d':' -f1) || {
echo "::warning title=Failed to match section end marker::End marker '$last' not found in input markdown; skipping '$f'..."
continue
}
# write out a line with the start-end range and filename
echo "${start},${end} ${f}" >> php-support.md.ed-unordered.txt
done
num_sections=$(set -o pipefail; wc -l php-support.md.ed-unordered.txt | awk '{ print $1 }')
(( $num_sections > 0 )) || echo "::warning title=No sections matched in input markdown::None of the generated sections coud be matched against the input markdown. No updates will occur."
echo "num_sections=${num_sections}" >> "$GITHUB_OUTPUT"
- name: Patch Dev Center article markdown
if: steps.find-section-markers.outputs.num_sections > 0
run: |
# init our ed script (https://www.gnu.org/software/diffutils/manual/html_node/Detailed-ed.html) for patching
echo -n > php-support.md.ed
# we now have the target file line ranges and source file names
# for patch to handle the line numbers in the ed script correctly, they must be ordered with the last changes coming first
# (otherwise every applied change will shift the line numbers for following changes)
sort -r -n -k1 -t',' php-support.md.ed-unordered.txt | while read range f; do
# write out an ed command that says "from starting line to ending line, replace with what follows"
echo "${range}c" >> php-support.md.ed
# write out new contents for range in command above
cat "$f" >> php-support.md.ed
# mark end of content
echo "." >> php-support.md.ed
done
patch --backup --ed php-support.md php-support.md.ed
- name: Dump diff of markdown contents
id: diff
if: steps.find-section-markers.outputs.num_sections > 0
run: |
# diff exits 0 if there are no differences, 1 if there are, 2 if there was trouble
# our patch --ed earlier added a newline at the end of the file
# the downloaded markdown, however, may not have a newline at the end of the file
# as a result, `diff` may produce a difference for that, but we want to ignore it
# we run through a sed that matches on end of file ($), then appends (a) nothing (after backslash)
# GNU sed will then add a newline at the end if there isn't one
diff -u <(sed -e '$a\' php-support.md.orig) <(sed -e '$a\' php-support.md) > php-support.diff && {
echo "::notice title=No diff in markdown::There were no differences after applying the generated sections to the input markdown."
echo "diff_result=0" >> "$GITHUB_OUTPUT"
} || {
diff_result=$?
echo "diff_result=${diff_result}" >> "$GITHUB_OUTPUT"
(( diff_result != 1 )) && {
echo "::error title=Unexpected error during diffing::Exit status of 'diff' command was '${diff_result}'."
exit ${diff_result}
}
echo '## Diff of changes to ["PHP Support" Dev Center article](https://devcenter.heroku.com/articles/php-support)' >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == true && '> [!IMPORTANT]' || '-n' }}" >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == true && '> **This is based on the source bucket (due to dry-run mode)**, not the destination bucket.' || '-n' }}" >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == false && '-n' || '' }}" >> "$GITHUB_STEP_SUMMARY"
echo '``````diff' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
cat php-support.diff >> "$GITHUB_STEP_SUMMARY"
echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
}
- name: Upload diff as artifact
if: steps.find-section-markers.outputs.num_sections > 0 && steps.diff.outputs.diff_result == 1
uses: actions/upload-artifact@v4
with:
name: devcenter-diff
path: |
have_crlf.txt
php-support.diff
devcenter-update:
needs: devcenter-generate
if: ${{ needs.devcenter-generate.outputs.diff_result == 1 }}
runs-on: ubuntu-24.04
env:
HEROKU_DEVCENTER_API_TOKEN: ${{ secrets.HEROKU_DEVCENTER_API_TOKEN }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dos2unix
run: |
sudo apt-get update
sudo apt-get install dos2unix
- name: Download current Dev Center article markdown
run: |
set -o pipefail
# we need to query this JSON twice, and a temp file is easiest for error handling
curl -H "Accept: application/json" https://devcenter.heroku.com/api/v1/articles/php-support > php-support.json
# jq -j, not -r, otherwise we get a stray trailing newline
cat php-support.json | jq -je '.content' > php-support.md
# here we want the trailing newline; run through alternative operator and empty filter to cause exit status 4 with -e if field missing
cat php-support.json | jq -re '.id // empty | @sh "article_id=\(.)"' >> "$GITHUB_ENV"
# Because the articles are edited in a web interface, they likely use CRLF line endings.
# We will be patching using an LF diff.
# For this reason, we may have to convert to LF, so we check if the file would be converted by dos2unix using the --info option.
# The "c" info flag prints only file names that would trigger conversion; we first remember this output for the next step via tee -a.
# The "0" flag triggers zero-byte output for happy consumption by xargs
# Then, we finally run the conversion (if needed) by passing the file name to dos2unix again via xargs.
dos2unix --info=c0 php-support.md | tee have_crlf.txt | xargs -r0 dos2unix
- name: Download diff artifact
uses: actions/download-artifact@v4
with:
name: devcenter-diff
path: devcenter-diffs
- name: Patch Dev Center article markdown
run: |
patch php-support.md devcenter-diffs/php-support.diff
# convert back to the original CRLF if dos2unix ran in an earlier step (have_crlf.txt will be empty if not, and xargs will not run due to -r)
cat have_crlf.txt | xargs -r0 unix2dos
- name: Validate Dev Center article
if: inputs.dry-run == true || inputs.update-devcenter == false
run: |
set -o pipefail
curl -sS --fail-with-body -X POST -K <(echo -n "user = bot:"; printenv "HEROKU_DEVCENTER_API_TOKEN") -H "Accept: application/json" --data-urlencode 'article[content]@php-support.md' "https://devcenter.heroku.com/api/v1/private/articles/${article_id}/validate.json"
- name: Output Dev Center article
if: inputs.dry-run == true || inputs.update-devcenter == false
run: |
echo '## Updated markdown for ["PHP Support" Dev Center article](https://devcenter.heroku.com/articles/php-support)' >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == true && '> [!IMPORTANT]' || '-n' }}" >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == true && '> **This is based on the source bucket** (due to dry-run mode), not the destination bucket.' || '-n' }}" >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == false && '-n' || '' }}" >> "$GITHUB_STEP_SUMMARY"
echo "> [!WARNING]" >> "$GITHUB_STEP_SUMMARY"
echo "> **Dev Center has not been updated with the contents below**${{ inputs.dry-run == true && ' (due to dry-run mode)' || ', copy the Markdown for manual updating!' }}" >> "$GITHUB_STEP_SUMMARY"
echo >> "$GITHUB_STEP_SUMMARY"
echo '``````markdown' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
cat php-support.md >> "$GITHUB_STEP_SUMMARY"
echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
- name: Update Dev Center article
if: inputs.dry-run == false && inputs.update-devcenter == true
run: |
set -o pipefail
curl -sS --fail-with-body -X PUT -K <(echo -n "user = bot:"; printenv "HEROKU_DEVCENTER_API_TOKEN") -H "Accept: application/json" --data-urlencode 'article[content]@php-support.md' "https://devcenter.heroku.com/api/v1/private/articles/${article_id}.json"
echo 'Successfully updated ["PHP Support" Dev Center article](https://devcenter.heroku.com/articles/php-support) with synced packages.' >> "$GITHUB_STEP_SUMMARY"
changelog-generate:
needs: sync
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install PHP and Composer
uses: shivammathur/setup-php@v2
with:
php-version: "8.2"
tools: "composer:2.8"
- name: Install Dev Center generator dependencies
run: |
composer install -d support/devcenter/
- name: Download all sync log artifacts
uses: actions/download-artifact@v4
with:
path: synclogs
pattern: synclog-*
merge-multiple: true
- name: Generate Changelog markdown
run: |
set -o pipefail
# concatenate all sync logs and feed them to changelog.php
# immediately split changelog title and body (separated by a "----" line) for easier copy/paste
# this also lets us detect if there even is anything to output in the next step (there won't be a changelog-01.md if the changelog is empty)
# the '{*}' pattern works around a bug in older coreutils csplits: https://github.com/coreutils/coreutils/commit/7cf45f4f6a093a927d3139c87f52999dd2c750ec
cat synclogs/sync-*.log | support/devcenter/changelog.php | csplit -s -z -f 'changelog-' -b '%02d.md' --suppress-matched - '/^----$/' '{*}'
- name: Output Changelog markdown
if: ${{ hashFiles('changelog-01.md') != '' }}
run: |
shopt -s extglob
echo '## Markdown for package Changelog entry' >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == true && '> [!WARNING]' || '-n' }}" >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == true && '> **These changes have not been synced to the destination bucket**, the changelog entry is for reference only!' || '-n' }}" >> "$GITHUB_STEP_SUMMARY"
echo "${{ inputs.dry-run == false && '-n' || '' }}" >> "$GITHUB_STEP_SUMMARY"
echo '### Title' >> "$GITHUB_STEP_SUMMARY"
echo '``````markdown' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
cat changelog-00.md >> "$GITHUB_STEP_SUMMARY"
echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
echo '### Content' >> "$GITHUB_STEP_SUMMARY"
echo '``````markdown' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks
cat changelog-!(00).md >> "$GITHUB_STEP_SUMMARY"
echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks