From 9a3324328fe4cd1aa5a2fa59b4ee3f7745d75c9b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 7 Dec 2023 12:21:04 -0600 Subject: [PATCH] fix: move job runner to /etc/cron.d; improve elastic-rebuild --- src/roles/cron/tasks/main.yml | 12 ++--- .../cron/templates/meza-ansible.crontab.j2 | 53 ------------------- .../templates/run-jobs-and-custom-crons.j2 | 42 +++++++++++++++ .../templates/elastic-build-index.sh.j2 | 10 ++-- .../templates/elastic-rebuild-all.sh.j2 | 53 +++++++++++++++++++ 5 files changed, 107 insertions(+), 63 deletions(-) delete mode 100644 src/roles/cron/templates/meza-ansible.crontab.j2 create mode 100644 src/roles/cron/templates/run-jobs-and-custom-crons.j2 diff --git a/src/roles/cron/tasks/main.yml b/src/roles/cron/tasks/main.yml index b3f323e1d..7814bd152 100644 --- a/src/roles/cron/tasks/main.yml +++ b/src/roles/cron/tasks/main.yml @@ -12,13 +12,13 @@ state: started enabled: yes -- name: Ensure crontab file up-to-date +- name: Ensure runjobs/custom-crons cron file in place template: - src: meza-ansible.crontab.j2 - dest: "{{ m_home }}/meza-ansible/.ansible-cronfile" - owner: meza-ansible - group: wheel - mode: 0755 + src: "run-jobs-and-custom-crons.j2" + dest: "/etc/cron.d/run-jobs-and-custom-crons-{{ env }}" + owner: "{{ m_crond_owner }}" + group: "{{ m_crond_group }}" + mode: "{{ m_crond_mode }}" - name: Ensure runAllJobs.php in place template: diff --git a/src/roles/cron/templates/meza-ansible.crontab.j2 b/src/roles/cron/templates/meza-ansible.crontab.j2 deleted file mode 100644 index badc3efc3..000000000 --- a/src/roles/cron/templates/meza-ansible.crontab.j2 +++ /dev/null @@ -1,53 +0,0 @@ -# {{ ansible_managed }} -# -# crontab for meza - -{% if 'logging_servers' in groups and inventory_hostname in groups['logging_servers'] and server_performance_crontime %} -# -# Cron job for logging server performance info -# -{{ server_performance_crontime }} {{ m_deploy }}/server-performance.sh -{% endif %} - -{% if disk_space_usage_mount_name is defined and 'logging_servers' in groups and inventory_hostname in groups['logging_servers'] %} -# -# Check disk space usage -# -{{ disk_space_usage_crontime }} {{ m_deploy }}/disk-space-usage.sh -{% endif %} - -{% if inventory_hostname in groups['app_servers'] %} -# -# Run a small set of jobs across all wikis periodically -# -# run_jobs_freq_maxtime run_jobs_freq_totalmaxtime run_jobs_freq_maxjobs run_jobs_freq_maxload -{{ run_jobs_freq_crontime }} WIKI={{ list_of_wikis[0] }} php {{ m_deploy }}/runAllJobs.php {{ run_jobs_freq_maxtime }} {{ run_jobs_freq_totalmaxtime }} {{ run_jobs_freq_maxjobs }} {{ run_jobs_freq_maxload }} >> {{ m_logs }}/jobqueue/cron_runAllJobs_`date "+\%Y\%m\%d"`.log 2>&1 - -# -# Run all jobs on all wikis -# Note: WIKI= does not matter which wiki. Just needs one to load -# settings. -# -{{ run_all_jobs_crontime }} WIKI={{ list_of_wikis[0] }} php {{ m_deploy }}/runAllJobs.php >> {{ m_logs }}/jobqueue/cron_runAllJobs_`date "+\%Y\%m\%d"`.log 2>&1 - -{% endif %} - -{% if inventory_hostname in groups['app_servers'] %} -# -# Clean out uploads temporary files nightly -# -{{ clean_upload_stash_crontime }} sudo meza maint cleanuploadstash {{ env }} >> {{ m_logs }}/cleanup/uploadstash_`date "+\%Y-\%m"`.log 2>&1 - -{% endif %} - -{% if custom_crons is defined %} -# -# The following jobs are custom definitions from this meza-instance's config -# -{% for cron in custom_crons %}{% if inventory_hostname in groups[cron.server_type] %} -{{ cron.time }} {{ cron.job }} - -{% endif %}{% endfor %} - -# END custom crons -{% endif %} diff --git a/src/roles/cron/templates/run-jobs-and-custom-crons.j2 b/src/roles/cron/templates/run-jobs-and-custom-crons.j2 new file mode 100644 index 000000000..250d1260b --- /dev/null +++ b/src/roles/cron/templates/run-jobs-and-custom-crons.j2 @@ -0,0 +1,42 @@ +# {{ ansible_managed }} +# +# crontab for meza run-jobs and custom-crons for environment "{{ env }}" +SHELL=/bin/bash +PATH=/sbin:/bin:/usr/sbin:/usr/bin +MAILTO=root + +{% if inventory_hostname in groups['app_servers'] %} +# +# Run a small set of jobs across all wikis periodically +# +# run_jobs_freq_maxtime run_jobs_freq_totalmaxtime run_jobs_freq_maxjobs run_jobs_freq_maxload +{{ run_jobs_freq_crontime }} root WIKI={{ list_of_wikis[0] }} php {{ m_deploy }}/runAllJobs.php {{ run_jobs_freq_maxtime }} {{ run_jobs_freq_totalmaxtime }} {{ run_jobs_freq_maxjobs }} {{ run_jobs_freq_maxload }} >> {{ m_logs }}/jobqueue/cron_runAllJobs_`date "+\%Y\%m\%d"`.log 2>&1 + +# +# Run all jobs on all wikis +# Note: WIKI= does not matter which wiki. Just needs one to load +# settings. +# +{{ run_all_jobs_crontime }} root WIKI={{ list_of_wikis[0] }} php {{ m_deploy }}/runAllJobs.php >> {{ m_logs }}/jobqueue/cron_runAllJobs_`date "+\%Y\%m\%d"`.log 2>&1 + +{% endif %} + +{% if inventory_hostname in groups['app_servers'] %} +# +# Clean out uploads temporary files nightly +# +{{ clean_upload_stash_crontime }} root meza maint cleanuploadstash {{ env }} >> {{ m_logs }}/cleanup/uploadstash_`date "+\%Y-\%m"`.log 2>&1 + +{% endif %} + +{% if custom_crons is defined %} +# +# The following jobs are custom definitions from this meza-instance's config +# +{% for cron in custom_crons %}{% if inventory_hostname in groups[cron.server_type] %} +{{ cron.time }} root {{ cron.job }} + +{% endif %}{% endfor %} + +# END custom crons +{% endif %} \ No newline at end of file diff --git a/src/roles/mediawiki/templates/elastic-build-index.sh.j2 b/src/roles/mediawiki/templates/elastic-build-index.sh.j2 index 4b5c75035..049329208 100644 --- a/src/roles/mediawiki/templates/elastic-build-index.sh.j2 +++ b/src/roles/mediawiki/templates/elastic-build-index.sh.j2 @@ -1,9 +1,7 @@ echo "******* Generating elasticsearch index *******" -disable_search_file="{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php" - -# disable search update in wiki-specific settings -echo -e " "$disable_search_file" +# Remove any broken cirrus update jobs +mysql "wiki_$wiki_id" -e "DELETE FROM job WHERE job_cmd = 'cirrusSearchElasticaWrite'" # Run script to generate elasticsearch index cd "{{ m_mediawiki }}" @@ -12,6 +10,10 @@ WIKI="$wiki_id" php "{{ m_mediawiki }}/extensions/CirrusSearch/maintenance/Updat # Remove search-update disable in wiki-specific settings rm -f "$disable_search_file" +# Restart PHP/Apache to ensure ^ change is picked up +systemctl restart httpd +systemctl restart php-fpm + # Bootstrap the search index # # Note that this can take some time diff --git a/src/roles/mediawiki/templates/elastic-rebuild-all.sh.j2 b/src/roles/mediawiki/templates/elastic-rebuild-all.sh.j2 index 214fbaa55..91b3b3cc7 100644 --- a/src/roles/mediawiki/templates/elastic-rebuild-all.sh.j2 +++ b/src/roles/mediawiki/templates/elastic-rebuild-all.sh.j2 @@ -9,6 +9,47 @@ fi wiki_dir="{{ m_htdocs }}/wikis" cd "$wiki_dir" + +# Before processing any wikis, stop search indexing on all. We're gonna nuke the whole index, across +# all wikis before starting indexing, and we don't want any wikis to try to index anything until +# their proper index is recreated. +for d in $do_wikis; do + + if [ -z "$1" ]; then + wiki_id=${d%/} + else + wiki_id="$d" + fi + + if [ ! -d "$wiki_dir/$wiki_id" ]; then + echo "\"$wiki_id\" not a valid wiki ID" + continue + fi + + # Disable search indexing in wiki-specific settings, and tell the + # wiki to not use CirrusSearch for now by nullifying $wgSearchType. + echo -e " "{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php" + echo -e "\$wgDisableSearchUpdate = true;" >> "{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php" + echo -e "\$wgSearchType = null;" >> "{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php" + + # Restart PHP/Apache to ensure ^ change is picked up + systemctl restart httpd + systemctl restart php-fpm + +done + +# Print elasticsearch indexes +curl -X GET 'http://localhost:9200/_cat/indices?v' + +# Nuke all the indexes +curl -X DELETE 'http://localhost:9200/_all' + +# Print again (should be empty) +curl -X GET 'http://localhost:9200/_cat/indices?v' + +# Create metastore index (used by all wikis) +sudo WIKI="$wiki_id" php Metastore.php --upgrade + for d in $do_wikis; do if [ -z "$1" ]; then @@ -49,4 +90,16 @@ for d in $do_wikis; do echo "elastic-build-index completed for \"$wiki_id\" at $endtimestamp" fi + # Run all the jobs for this wiki + maxjobs=1000 + while [ $(WIKI="$wiki_id" php /opt/htdocs/mediawiki/maintenance/showJobs.php) -gt 0 ]; do + WIKI="$wiki_id" php /opt/htdocs/mediawiki/maintenance/runJobs.php --maxjobs="$maxjobs" + echo + echo "Up to 1000 jobs complete. Pausing for 5 seconds." + echo + sleep 5 + done; + done + +