fix: move job runner to /etc/cron.d; improve elastic-rebuild

src/roles/cron/tasks/main.yml

@@ -12,13 +12,13 @@
     state: started
     enabled: yes
 
-- name: Ensure crontab file up-to-date
+- name: Ensure runjobs/custom-crons cron file in place
   template:
-    src: meza-ansible.crontab.j2
-    dest: "{{ m_home }}/meza-ansible/.ansible-cronfile"
-    owner: meza-ansible
-    group: wheel
-    mode: 0755
+    src: "run-jobs-and-custom-crons.j2"
+    dest: "/etc/cron.d/run-jobs-and-custom-crons-{{ env }}"
+    owner: "{{ m_crond_owner }}"
+    group: "{{ m_crond_group }}"
+    mode: "{{ m_crond_mode }}"
 
 - name: Ensure runAllJobs.php in place
   template:

src/roles/cron/templates/run-jobs-and-custom-crons.j2

@@ -0,0 +1,42 @@
+# {{ ansible_managed }}
+#
+# crontab for meza run-jobs and custom-crons for environment "{{ env }}"
+SHELL=/bin/bash
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+MAILTO=root
+
+{% if inventory_hostname in groups['app_servers'] %}
+#
+# Run a small set of jobs across all wikis periodically
+#
+# run_jobs_freq_maxtime run_jobs_freq_totalmaxtime run_jobs_freq_maxjobs run_jobs_freq_maxload
+{{ run_jobs_freq_crontime }} root WIKI={{ list_of_wikis[0] }} php {{ m_deploy }}/runAllJobs.php {{ run_jobs_freq_maxtime }} {{ run_jobs_freq_totalmaxtime }} {{ run_jobs_freq_maxjobs }} {{ run_jobs_freq_maxload }} >> {{ m_logs }}/jobqueue/cron_runAllJobs_`date "+\%Y\%m\%d"`.log 2>&1
+
+#
+# Run all jobs on all wikis
+# Note: WIKI=<wiki_id> does not matter which wiki. Just needs one to load
+# settings.
+#
+{{ run_all_jobs_crontime }} root WIKI={{ list_of_wikis[0] }} php {{ m_deploy }}/runAllJobs.php >> {{ m_logs }}/jobqueue/cron_runAllJobs_`date "+\%Y\%m\%d"`.log 2>&1
+
+{% endif %}
+
+{% if inventory_hostname in groups['app_servers'] %}
+#
+# Clean out uploads temporary files nightly
+#
+{{ clean_upload_stash_crontime }} root meza maint cleanuploadstash {{ env }} >> {{ m_logs }}/cleanup/uploadstash_`date "+\%Y-\%m"`.log 2>&1
+
+{% endif %}
+
+{% if custom_crons is defined  %}
+#
+# The following jobs are custom definitions from this meza-instance's config
+#
+{% for cron in custom_crons %}{% if inventory_hostname in groups[cron.server_type] %}
+{{ cron.time }} root {{ cron.job }}
+
+{% endif %}{% endfor %}
+
+# END custom crons
+{% endif %}

src/roles/mediawiki/templates/elastic-build-index.sh.j2

@@ -1,9 +1,7 @@
 echo "******* Generating elasticsearch index *******"
 
-disable_search_file="{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php"
-
-# disable search update in wiki-specific settings
-echo -e "<?php\n\$wgDisableSearchUpdate = true;\n" > "$disable_search_file"
+# Remove any broken cirrus update jobs
+mysql "wiki_$wiki_id" -e "DELETE FROM job WHERE job_cmd = 'cirrusSearchElasticaWrite'"
 
 # Run script to generate elasticsearch index
 cd "{{ m_mediawiki }}"
@@ -12,6 +10,10 @@ WIKI="$wiki_id" php "{{ m_mediawiki }}/extensions/CirrusSearch/maintenance/Updat
 # Remove search-update disable in wiki-specific settings
 rm -f "$disable_search_file"
 
+# Restart PHP/Apache to ensure ^ change is picked up
+systemctl restart httpd
+systemctl restart php-fpm
+
 # Bootstrap the search index
 #
 # Note that this can take some time

src/roles/mediawiki/templates/elastic-rebuild-all.sh.j2

@@ -9,6 +9,47 @@ fi
 wiki_dir="{{ m_htdocs }}/wikis"
 
 cd "$wiki_dir"
+
+# Before processing any wikis, stop search indexing on all. We're gonna nuke the whole index, across
+# all wikis before starting indexing, and we don't want any wikis to try to index anything until
+# their proper index is recreated.
+for d in $do_wikis; do
+
+	if [ -z "$1" ]; then
+		wiki_id=${d%/}
+	else
+		wiki_id="$d"
+	fi
+
+	if [ ! -d "$wiki_dir/$wiki_id" ]; then
+		echo "\"$wiki_id\" not a valid wiki ID"
+		continue
+	fi
+
+	# Disable search indexing in wiki-specific settings, and tell the
+	# wiki to not use CirrusSearch for now by nullifying $wgSearchType.
+	echo -e "<?php"                           >  "{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php"
+	echo -e "\$wgDisableSearchUpdate = true;" >> "{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php"
+	echo -e "\$wgSearchType = null;"          >> "{{ m_deploy }}/public/wikis/$wiki_id/postLocalSettings.d/disable-search-update.php"
+
+	# Restart PHP/Apache to ensure ^ change is picked up
+	systemctl restart httpd
+	systemctl restart php-fpm
+
+done
+
+# Print elasticsearch indexes
+curl -X GET 'http://localhost:9200/_cat/indices?v'
+
+# Nuke all the indexes
+curl -X DELETE 'http://localhost:9200/_all'
+
+# Print again (should be empty)
+curl -X GET 'http://localhost:9200/_cat/indices?v'
+
+# Create metastore index (used by all wikis)
+sudo WIKI="$wiki_id" php Metastore.php --upgrade
+
 for d in $do_wikis; do
 
 	if [ -z "$1" ]; then
@@ -49,4 +90,16 @@ for d in $do_wikis; do
 		echo "elastic-build-index completed for \"$wiki_id\" at $endtimestamp"
 	fi
 
+	# Run all the jobs for this wiki
+	maxjobs=1000
+	while [ $(WIKI="$wiki_id" php /opt/htdocs/mediawiki/maintenance/showJobs.php) -gt 0 ]; do
+		WIKI="$wiki_id" php /opt/htdocs/mediawiki/maintenance/runJobs.php --maxjobs="$maxjobs"
+		echo
+		echo "Up to 1000 jobs complete. Pausing for 5 seconds."
+		echo
+		sleep 5
+	done;
+
 done
+
+