progress on on-schedule!

.ci/on-commit.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e -o pipefail
 
 # This script is used to determine which packages to build based on the recent commits and run necessary checks

.ci/on-schedule.sh

@@ -1,5 +1,168 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e -o pipefail
 
 # This script is triggered by a scheduled pipeline
 
+# Check if the scheduled tag does not exist or scheduled does not point to HEAD
+if ! [ "$(git tag -l "scheduled")" ] || [ "$(git rev-parse HEAD)" != "$(git rev-parse scheduled)" ]; then
+    echo "Previous on-commit pipeline did not seem to run successfully. Aborting." >&2
+    exit 1
+fi
+
+source .ci/util.shlib
+
+TMPDIR="${TMPDIR:-/tmp}"
+
+PACKAGES=()
+MODIFIED_PACKAGES=()
+UTIL_GET_PACKAGES PACKAGES
+
+# $1: dir1
+# $2: dir2
+function package_changed() {
+    # Check if the package has changed
+    # NOTE: Pay attention! Okay, *snaps fingers*. We don't care if anything but the PKGBUILD or .SRCINFO has changed.
+    # Any properly built PKGBUILD will use hashes, which will change
+    if diff -q "$1/PKGBUILD" "$2/PKGBUILD" >/dev/null; then
+        if [ ! -f "$1/.SRCINFO" ] && [ ! -f "$2/.SRCINFO" ]; then
+            return 1
+        elif if [ -f "$1/.SRCINFO" ] && [ -f "$2/.SRCINFO" ]; then
+            if diff -q "$1/.SRCINFO" "$2/.SRCINFO" >/dev/null; then
+                return 1
+            fi
+        fi
+    fi
+    return 0
+}
+
+# $1: VARIABLES
+# $2: git URL
+function update_via_git() {
+    local -n VARIABLES="${1:-VARIABLES}"
+    local pkgbase="${VARIABLES[PKGBASE]}"
+
+    git clone --depth=1 "$2" "$TMPDIR/aur-pulls/$pkgbase"
+
+    # We always run shfmt on the PKGBUILD. Two runs of shfmt on the same file should not change anything
+    shfmt -w "$TMPDIR/aur-pulls/$pkgbase/PKGBUILD"
+
+    if package_changed; then
+        # Rsync: delete files in the destination that are not in the source. Exclude deleting .CI_CONFIG, exclude copying .git
+        rsync -a --delete --exclude=.CI_CONFIG --exclude=.git "$TMPDIR/aur-pulls/$pkgbase/" "$pkgbase/"
+        MODIFIED_PACKAGES+=("$pkgbase")
+    fi
+}
+
+# $1: VARIABLES
+# $2: new timestamp
+function update_aur_timestamp() {
+    local -n VARIABLES="${1:-VARIABLES}"
+    local new_timestamp="$2"
+
+    if [ "$new_timestamp" != "0" ]; then
+        VARIABLES[CI_PKGBUILD_TIMESTAMP]="$new_timestamp"
+    fi
+}
+
+function update_pkgbuild() {
+    local -n VARIABLES="${1:-VARIABLES}"
+    local pkgbase="${VARIABLES[PKGBASE]}"
+    if ! [ -v "VARIABLES[CI_PKGBUILD_SOURCE]" ]; then
+        return 0
+    fi
+
+    local PKGBUILD_SOURCE="${VARIABLES[CI_PKGBUILD_SOURCE]}"
+
+    # Check if format is aur:pkgbase
+    if [[ "$PKGBUILD_SOURCE" != aur:* ]]; then
+        update_via_git VARIABLES "$PKGBUILD_SOURCE"
+    else
+        local pkgbase="${PKGBUILD_SOURCE#aur:}"
+        local git_url="https://aur.archlinux.org/${pkgbase}.git"
+
+        local NEW_TIMESTAMP
+        NEW_TIMESTAMP="$(curl -s "https://aur.archlinux.org/rpc/v5/info?arg[]=$pkgbase" | jq -r '.results[0].LastModified' || echo "0")"
+
+        # Check if CI_PKGBUILD_TIMESTAMP is set
+        if [ -v "VARIABLES[CI_PKGBUILD_TIMESTAMP]" ]; then
+            local PKGBUILD_TIMESTAMP="${VARIABLES[CI_PKGBUILD_TIMESTAMP]}"
+            if [ "$PKGBUILD_TIMESTAMP" != "$NEW_TIMESTAMP" ]; then
+                update_via_git VARIABLES "$git_url"
+                update_aur_timestamp VARIABLES "$NEW_TIMESTAMP"
+            fi
+        else
+            update_via_git VARIABLES "$git_url"
+            update_aur_timestamp VARIABLES "$NEW_TIMESTAMP"
+        fi
+    fi
+}
+
+# $1: VARIABLES
+# $2: new commit
+function update_vcs_commit() {
+    local -n VARIABLES="${1:-VARIABLES}"
+    local new_commit="$2"
+
+    if [ "$new_commit" != "0" ]; then
+        VARIABLES[CI_GIT_COMMIT]="$new_commit"
+    fi
+}
+
+function update_vcs() {
+    local -n VARIABLES="${1:-VARIABLES}"
+    local pkgbase="${VARIABLES[PKGBASE]}"
+
+    # Check if pkgbase ends with -git
+    if [[ "$pkgbase" != *-git ]]; then
+        return 0
+    fi
+
+    # Check if .SRCINFO exists. We can't work with a -git package without it
+    if ! [ -f "$pkgbase/.SRCINFO" ]; then
+        return 0
+    fi
+
+    # Parse the first source from the .SRCINFO file
+    local source
+    source=$(grep -m 1 -oP '\ssource\s=\s.*git\+\K.*$' "$pkgbase/.SRCINFO")
+
+    if [ -z "$source" ]; then
+        return 0
+    fi
+
+    local _NEWEST_COMMIT
+    _NEWEST_COMMIT="$(git ls-remote "$_SOURCE" | grep -m1 -oP '\w+(?=\tHEAD)' || echo "0")"
+
+    # Check if CI_GIT_COMMIT is set
+    if [ -v "VARIABLES[CI_GIT_COMMIT]" ]; then
+        local CI_GIT_COMMIT="${VARIABLES[CI_GIT_COMMIT]}"
+        if [ "$CI_GIT_COMMIT" != "$_NEWEST_COMMIT" ]; then
+            update_vcs_commit "$_NEWEST_COMMIT"
+            MODIFIED_PACKAGES+=("$pkgbase")
+        fi
+    else
+        update_vcs_commit "$_NEWEST_COMMIT"
+        MODIFIED_PACKAGES+=("$pkgbase")
+    fi
+}
+
+mkdir "$TMPDIR/aur-pulls"
+
+# Loop through all packages
+for package in "${PACKAGES[@]}"; do
+    declare -A VARIABLES
+    if UTIL_READ_MANAGED_PACAKGE "$package" VARIABLES; then
+        update_pkgbuild VARIABLES
+        update_vcs VARIABLES
+    fi
+done
+
+if ! git diff --exit-code --quiet; then
+    git add .
+    git commit -m "chore(packages): update packages [skip ci]"
+fi
+
+"$(dirname "$(realpath "$0")")"/schedule-packages.sh "${MODIFIED_PACKAGES[*]}"
+
+git tag -f scheduled
+git push --atomic "$REPO_URL" HEAD:main refs/tags/scheduled

.ci/schedule-packages.sh

@@ -1,14 +1,17 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 
 # This script parses the parameters passed to this script and outputs a list of package names to a file
 
 mapfile -t PACKAGES <<< "$@"
 
+source .ci/util.shlib
+
 if [ -v "PACKAGES[0]" ] && [ "${PACKAGES[0]}" == "all" ]; then
     echo "Rebuild of all packages requested."
-    mapfile -t PACKAGES < <(find . -mindepth 1 -type d -not -path '*/.*' -printf '%P\n')
+    local PACKAGES
+    UTIL_GET_PACKAGES PACKAGES
 fi
 
 # Check if the array of packages is empty

.ci/util.shlib

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+KNOWN_VARIABLE_LIST=(CI_PKGBUILD_SOURCE CI_GIT_COMMIT CI_PKGBUILD_TIMESTAMP)
+
+# Get a list of all the packages in the repo
+function UTIL_GET_PACKAGES() {
+    local -n nameref_array=${1:-PACKAGES}
+    mapfile -t nameref_array < <(find . -mindepth 1 -type d -not -path '*/.*' -printf '%P\n')
+}
+
+function UTIL_PRUNE_UNKNOWN_VARIABLES() {
+    local -n nameref_assoc_array="${1:-VARIABLES}"
+    for key in "${!nameref_assoc_array[@]}"; do
+        if [[ ! " ${KNOWN_VARIABLE_LIST[@]} " =~ " ${key} " ]]; then
+            unset nameref_assoc_array[$key]
+        fi
+    done
+}
+
+function UTIL_READ_VARIABLES_FROM_FILE() {
+    local file=$1
+    local -n nameref_assoc_array="${2:-VARIABLES}"
+    echo $file
+    while IFS= read -r line || [ -n "$line" ]; do
+        if [[ "$line" =~ ^[[:space:]]*([a-zA-Z0-9_]+)[[:space:]]*=[[:space:]]*(.*)[[:space:]]*$ ]]; then
+            nameref_assoc_array["${BASH_REMATCH[1]}"]="${BASH_REMATCH[2]}"
+        fi
+    done < "$file"
+}
+
+function UTIL_WRITE_VARIABLES_TO_FILE() {
+    local file=$1
+    local -n nameref_assoc_array="${2:-VARIABLES}"
+    for key in "${!nameref_assoc_array[@]}"; do
+        echo "$key=${nameref_assoc_array[$key]}" >> "$file"
+    done
+}
+
+function UTIL_READ_MANAGED_PACAKGE() {
+    local target_file="./${1}/.CI_CONFIG"
+    if [ -f "$target_file" ]; then
+        local -n nameref_assoc_array="${2:-VARIABLES}"
+        UTIL_READ_VARIABLES_FROM_FILE "$target_file" nameref_assoc_array
+
+        # Check if any variable at all was read
+        if [ ${#nameref_assoc_array[@]} -ne 0 ]; then
+            nameref_assoc_array[PKGBASE]="$1"
+            UTIL_PRUNE_UNKNOWN_VARIABLES nameref_assoc_array
+            return 0
+        fi
+    fi
+    return 1
+}

.github/workflows/on-schedule.yml

@@ -11,11 +11,15 @@ env:
     REDIS_SSH_PORT: 400
     REDIS_SSH_USER: package-deployer
     REPO_NAME: chaotic-aur
+    CI_HUMAN_REVIEW: false
 
 # Controls when the workflow will run
 on:
-  push:
-    branches: [ "main" ]
+  schedule:
+    # Hourly
+    - cron: '0 * * * *'
+  # Manually
+  workflow_dispatch:
 
 concurrency:
   group: chaotic
@@ -28,6 +32,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: awalsh128/cache-apt-pkgs-action@v1
+        with:
+          packages: shfmt jq rsync
+          version: 1.0
       - name: Execute on-schedule tasks...
         id: parse
         run: |

paru/.CI_CONFIG

@@ -0,0 +1 @@
+CI_PKGBUILD_SOURCE=aur:paru

paru/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Morgan <[email protected]>
 pkgname=paru
 pkgver=2.0.1
-pkgrel=2
+pkgrel=1
 pkgdesc='Feature packed AUR helper'
 url='https://github.com/morganamilo/paru'
 source=("$pkgname-$pkgver.tar.gz::https://github.com/Morganamilo/paru/archive/v$pkgver.tar.gz")

whoogle-git/.CI_CONFIG

@@ -0,0 +1 @@
+CI_PKGBUILD_SOURCE=aur:whoogle-git

whoogle-git/.SRCINFO

@@ -0,0 +1,24 @@
+pkgbase = whoogle-git
+	pkgdesc = A self-hosted, ad-free, privacy-respecting metasearch engine
+	pkgver = 0.8.4.r6.g7313edf
+	pkgrel = 1
+	url = https://github.com/benbusby/whoogle-search
+	install = whoogle.install
+	arch = x86_64
+	arch = aarch64
+	license = MIT
+	makedepends = git
+	depends = python
+	provides = whoogle
+	conflicts = whoogle
+	backup = etc/default/whoogle
+	source = git+https://github.com/benbusby/whoogle-search.git
+	source = whoogle.service
+	source = whoogle.conf
+	source = whoogle
+	sha256sums = SKIP
+	sha256sums = ab6256f3fdaac3ba58ddbb39bb5c24bde53312f0584ae4ed4ae74bc7752a07f4
+	sha256sums = 51cda92f3ad2166eb2cb63ff80561f48b39688a57b66291d2eee5e1c7fcd8ee3
+	sha256sums = e30ff5ecef199ce2a37b097709461c51ca07bdbbcc4609db74203834b62c60b1
+
+pkgname = whoogle-git

whoogle-git/.gitignore

@@ -0,0 +1,2 @@
+*.tar.gz
+*.tar.zst

whoogle-git/PKGBUILD

@@ -0,0 +1,48 @@
+# Maintainer: dr460nf1r3 <[email protected]>
+
+pkgname=whoogle-git
+pkgver=0.8.4.r6.g7313edf
+pkgrel=1
+pkgdesc='A self-hosted, ad-free, privacy-respecting metasearch engine'
+arch=(x86_64 aarch64)
+url="https://github.com/benbusby/whoogle-search"
+license=(MIT)
+depends=(python)
+makedepends=(git)
+provides=(whoogle)
+conflicts=(whoogle)
+backup=('etc/default/whoogle')
+source=("git+$url.git"
+	whoogle.service
+	whoogle.conf
+	whoogle)
+sha256sums=('SKIP'
+	'ab6256f3fdaac3ba58ddbb39bb5c24bde53312f0584ae4ed4ae74bc7752a07f4'
+	'51cda92f3ad2166eb2cb63ff80561f48b39688a57b66291d2eee5e1c7fcd8ee3'
+	'e30ff5ecef199ce2a37b097709461c51ca07bdbbcc4609db74203834b62c60b1')
+install=whoogle.install
+
+pkgver() {
+	cd whoogle-search
+	git describe --long --tags --abbrev=7 | sed 's/\([^-]*-g\)/r\1/;s/-/./g' | sed 's/v//g'
+}
+
+build() {
+	# Following official instructions
+	cd whoogle-search
+	python3 -m venv venv
+	source venv/bin/activate
+	pip install -r requirements.txt
+
+	# Cleanup unsed
+	rm -r .git .github docs test .dockerignore .gitignore .replit docker-compose.yml Dockerfile heroku.yml MANIFEST.in README.md requirements.txt
+}
+
+package() {
+	install -m0644 -D "$srcdir/whoogle" "$pkgdir/etc/default/whoogle"
+	install -m0644 -D "$srcdir/whoogle.conf" "$pkgdir/usr/lib/sysusers.d/whoogle.conf"
+	install -m0644 -D "$srcdir/whoogle.service" "$pkgdir/usr/lib/systemd/system/whoogle.service"
+	install -Dm0644 "$srcdir/whoogle-search/LICENSE" "$pkgdir/usr/share/licenses/whoogle-search/LICENSE"
+	install -dm0755 "$pkgdir/opt/whoogle-search"
+	cp -r "$srcdir/whoogle-search/" "$pkgdir/opt/"
+}

whoogle-git/whoogle

@@ -0,0 +1,2 @@
+BIND_ADDRESS=127.0.0.1
+LISTEN_PORT=5000

whoogle-git/whoogle.conf

@@ -0,0 +1 @@
+u whoogle - "Whoogle" /opt/whoogle-search

whoogle-git/whoogle.install

@@ -0,0 +1,14 @@
+post_install() {
+	mkdir -p /opt/whoogle-search/
+	chown -R whoogle:whoogle /opt/whoogle-search/
+	echo "Enable the systemd unit and visit localhost:5000 to start searching!"
+	echo "If the search engine can't be reached, execute 'chown -R whoogle:whoogle /opt/whoogle-search/' to fix permissions."
+}
+
+post_upgrade() {
+	chown -R whoogle:whoogle /opt/whoogle-search/
+}
+
+post_remove() {
+	echo "Configuration files are still present in /opt/whoogle-search, run 'sudo rm -r /opt/whoogle-search' to remove them."
+}