diff --git a/bin/coderay b/bin/coderay deleted file mode 100755 index 0370076..0000000 --- a/bin/coderay +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'coderay' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -bundle_binstub = File.expand_path("../bundle", __FILE__) -load(bundle_binstub) if File.file?(bundle_binstub) - -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("coderay", "coderay") diff --git a/bin/dromedary b/bin/dromedary deleted file mode 100755 index 942831a..0000000 --- a/bin/dromedary +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env ruby - -# Before doing anything else, need to -# force bundler to load up so we -# can require gems from the Gemfile -# -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) -bundle_binstub = File.expand_path("../bundle", __FILE__) - -if File.file?(bundle_binstub) - if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ - load(bundle_binstub) - else - abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. -Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") - end -end - -require "bundler/setup" - -# Add the local lib - -$LOAD_PATH.unshift (Pathname(__dir__).parent + 'lib').to_s -$LOAD_PATH.unshift (Pathname(__dir__).parent + 'indexer').to_s - -# Now we can actually load stuff - -require 'hanami/cli' -require "dromedary/services" -require 'med_installer' - -# Set this up as a CLI -module MedInstaller - class CLI - extend Hanami::CLI::Registry - end -end - - -# Let's simplify things: - -# Unzip and convert new data into a build directory -require "med_installer/prepare" -MedInstaller::CLI.register 'newdata prepare', MedInstaller::Prepare - -# Or do them one at a time -MedInstaller::CLI.register "newdata extract", MedInstaller::Extract -MedInstaller::CLI.register "newdata convert", MedInstaller::Convert - -require "med_installer/index_new_data" -MedInstaller::CLI.register 'newdata index_existing', MedInstaller::IndexNewData - -# Actually index -# Index things into solr -# MedInstaller::CLI.register 'extract', MedInstaller::Extract, aliases: ['unzip'] -# MedInstaller::CLI.register 'convert', MedInstaller::Convert, aliases: ['json'] -# MedInstaller::CLI.register 'index copy_from_build', MedInstaller::CopyFromBuild -# MedInstaller::CLI.register 'index entries', MedInstaller::Index::Entries -# MedInstaller::CLI.register 'index bib', MedInstaller::Index::Bib -# -# # Create the hyp_to_bibid.json mapping and put it in Dromedary.config.data_dir -MedInstaller::CLI.register 'index hyp_to_bibid', MedInstaller::Index::HypToBibID, aliases: ['mapping'] -# -# # Reload the solr config (necessary after a change to the solr config has been pushed) -# MedInstaller::CLI.register 'solr reload', MedInstaller::Solr::Reload, aliases: ['bounce'] -# -# # Rebuild the indexing underlying autocomplete (should never be necessary, but...) -MedInstaller::CLI.register 'solr rebuild_suggesters', MedInstaller::Solr::RebuildSuggesters -# -# -# # Remote deployment and execution -# # -# MedInstaller::CLI.register 'deploy', MedInstaller::Remote::Deploy -# -# # Restart removed from moku??? -# #MedInstaller::CLI.register 'restart', MedInstaller::Remote::Restart -# -# MedInstaller::CLI.register 'remote', MedInstaller::Remote::Dromedary -# MedInstaller::CLI.register 'remote-exec', MedInstaller::Remote::Exec -# -# -# # Maintenance mode -# MedInstaller::CLI.register 'maintenance_mode on', MedInstaller::Control::MaintenanceModeOn -# MedInstaller::CLI.register 'maintenance_mode off', MedInstaller::Control::MaintenanceModeOff -# -# # Solr utilities for your development environment. -# MedInstaller::CLI.register 'solr install', MedInstaller::Solr::Install -# #MedInstaller::CLI.register 'solr link', MedInstaller::Solr::Link # TODO: Fix this for solr libs -# MedInstaller::CLI.register 'solr start', MedInstaller::Solr::Start, aliases: ['restart'] -# MedInstaller::CLI.register 'solr stop', MedInstaller::Solr::Stop -# MedInstaller::CLI.register 'solr empty', MedInstaller::Solr::Empty, aliases: ['clear'] -# MedInstaller::CLI.register 'solr commit', MedInstaller::Solr::Commit -# MedInstaller::CLI.register 'solr optimize', MedInstaller::Solr::Optimize -# MedInstaller::CLI.register 'solr up', MedInstaller::Solr::Up, aliases: ['up?'] -# MedInstaller::CLI.register 'solr shell', MedInstaller::Solr::Shell - - - - -# MedInstaller::CLI.register 'newdata index', MedInstaller::IndexNewData -# MedInstaller::CLI.register 'ping_prometheus', MedInstaller::PingPrometheus -# MedInstaller::CLI.register 'extract_convert_index', MedInstaller::ExtractConvertIndex, aliases: ['load_new_data'] - -Hanami::CLI.new(MedInstaller::CLI).call - diff --git a/bin/htmldiff b/bin/htmldiff deleted file mode 100755 index cbb1d5d..0000000 --- a/bin/htmldiff +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'htmldiff' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -bundle_binstub = File.expand_path("../bundle", __FILE__) -load(bundle_binstub) if File.file?(bundle_binstub) - -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("diff-lcs", "htmldiff") diff --git a/bin/httpclient b/bin/httpclient deleted file mode 100755 index 07320f6..0000000 --- a/bin/httpclient +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'httpclient' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -bundle_binstub = File.expand_path("../bundle", __FILE__) -load(bundle_binstub) if File.file?(bundle_binstub) - -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("httpclient", "httpclient") diff --git a/bin/index_new_file.rb b/bin/index_new_file.rb index d865cbe..b83838d 100755 --- a/bin/index_new_file.rb +++ b/bin/index_new_file.rb @@ -31,6 +31,9 @@ require "dromedary/services" require "med_installer/indexing_steps" +# require "simplecov" +# SimpleCov.start + filename = ARGV.shift indexer = MedInstaller::IndexingSteps.new(zipfile: filename) diff --git a/bin/ldiff b/bin/ldiff deleted file mode 100755 index 198cc2b..0000000 --- a/bin/ldiff +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'ldiff' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -bundle_binstub = File.expand_path("../bundle", __FILE__) -load(bundle_binstub) if File.file?(bundle_binstub) - -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("diff-lcs", "ldiff") diff --git a/bin/nokogiri b/bin/nokogiri deleted file mode 100755 index f54cdc1..0000000 --- a/bin/nokogiri +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'nokogiri' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -bundle_binstub = File.expand_path("../bundle", __FILE__) -load(bundle_binstub) if File.file?(bundle_binstub) - -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("nokogiri", "nokogiri") diff --git a/compose.yml b/compose.yml index 7920d83..e5046ef 100644 --- a/compose.yml +++ b/compose.yml @@ -29,7 +29,7 @@ services: - RAILS_RELATIVE_URL_ROOT=/m/middle-english-dictionary - TZ=America/New_York - "DIRECT_URLS_TO_SOLR_REPLICAS=http://solr:8983 http://solr:8983" - - MANUALLY_BUILD_SUGGESTERS=true + - MANUALLY_BUILD_SUGGESTERS=false - PAUSE_TIME=10 env_file: - path: .app.env diff --git a/lib/med_installer.rb b/lib/med_installer.rb index 88b2afc..6ccdbed 100644 --- a/lib/med_installer.rb +++ b/lib/med_installer.rb @@ -3,17 +3,7 @@ require_relative "med_installer/logger" require_relative "med_installer/extract" -require_relative "med_installer/index" require_relative "med_installer/convert" -require_relative "med_installer/solr" require_relative "med_installer/indexer/entry_json_reader" require_relative "med_installer/indexer/bib_reader" -require_relative "med_installer/extract_convert_index" - -require_relative "med_installer/remote" -require_relative "med_installer/control" - -require_relative "med_installer/copy_from_build" -require_relative "med_installer/index_new_data" - require_relative "med_installer/ping_prometheus" diff --git a/lib/med_installer/control.rb b/lib/med_installer/control.rb deleted file mode 100644 index 147eb52..0000000 --- a/lib/med_installer/control.rb +++ /dev/null @@ -1,28 +0,0 @@ -require "hanami/cli" -require "annoying_utilities" -require_relative "logger" - -module MedInstaller - class Control - extend MedInstaller::Logger - - class MaintenanceModeOn < Hanami::CLI::Command - desc "Turn on maintenance mode (redirect all pages to temp down page)" - - def call(command) - File.open AnnoyingUtilities.maintenance_mode_flag_file, "w:utf-8" do |f| - f.puts "To take out of maintenance mode, remove this file manually - or by running `bin/dromedary maintenance_mode off`" - end - end - end - - class MaintenanceModeOff < Hanami::CLI::Command - desc "Turn off maintenance mode (redirect all pages to temp down page)" - - def call(command) - FileUtils.remove_file(AnnoyingUtilities.maintenance_mode_flag_file, :force) - end - end - end -end diff --git a/lib/med_installer/copy_from_build.rb b/lib/med_installer/copy_from_build.rb deleted file mode 100644 index ff78bec..0000000 --- a/lib/med_installer/copy_from_build.rb +++ /dev/null @@ -1,129 +0,0 @@ -require_relative "index" -require_relative "control" - -require "fileutils" - -require_relative "../../config/load_local_config" - -module MedInstaller - # Copy the already-munged files from the build directory into this - # instances data_dir, presumably for later indexing. - class CopyFromBuild < Hanami::CLI::Command - include MedInstaller::Logger - - # Error types for CopyFromBuild - class ConfigurationError < StandardError - end - - class ErrorWithFileList < StandardError - attr_accessor :files - - def initialize(msg, files: []) - super(msg) - @files = files - end - end - - class MissingDirectory < ErrorWithFileList - end - - class FilesTooOld < ErrorWithFileList - end - - class FileMissing < ErrorWithFileList - end - - class BuildFileMissing < ErrorWithFileList - end - - NEEDED_FILES = %w[entries.json.gz bib_all.xml hyp_to_bibid.json] - option :force, - required: false, - default: false, - values: %w[true false], - desc: "Force a copy even if the files in build aren't newer than those currently in this instance's data_dir" - - def call(**options) - @data_dir = AnnoyingUtilities.data_dir - @build_dir = Pathname.new(Dromedary.config.build_dir).realdirpath - @force = options.fetch(:force) - - validate_directories! - validate_build_files_exist! - - unless @force - validate_files_not_too_old! - end - - # We're finally ready. - - NEEDED_FILES.each do |f| - logger.info "Copying #{f} from #{@build_dir} to #{@data_dir}" - FileUtils.copy_file(build_file(f), current_file(f)) - end - rescue MissingDirectory, ConfigurationError, BuildFileMissing => e - error_with_file_list(e) - rescue FilesTooOld => e - msg = <= build_file(f).mtime - end - - unless too_old.empty? - raise FilesTooOld.new("Current files newer than build files", files: too_old) - end - end - - private - - def error_with_file_list(e) - logger.error "\n\n" + e.message + "[#{e.files.join(", ")}]" - raise e - end - - # Make sure a directory is defined and exists - def validate_dir!(dir:, label:) - raise ConfigurationError.new("#{label} is not configured") if dir.nil? - raise MissingDirectory.new("#{label} dir #{dir} does not exist. Aborting") unless dir.exist? - end - - # @return [Pathname] - def build_file(filename) - Pathname.new(@build_dir) + filename - end - - # @return [Pathname] - def current_file(filename) - Pathname.new(@data_dir) + filename - end - end -end diff --git a/lib/med_installer/extract_convert_index.rb b/lib/med_installer/extract_convert_index.rb deleted file mode 100644 index 5cce1e5..0000000 --- a/lib/med_installer/extract_convert_index.rb +++ /dev/null @@ -1,68 +0,0 @@ -require_relative "extract" -require_relative "convert" -require_relative "index" -require_relative "control" - -require "fileutils" - -require_relative "job_monitoring" - -require_relative "../../config/load_local_config" - -module MedInstaller - # Start from a zip file and go from there - class ExtractConvertIndex < Hanami::CLI::Command - include MedInstaller::Logger - - argument :zipfile, required: true, desc: "The path to the zipfile (downloaded from Box)" - - def call(zipfile:) - # metrics = MiddleEnglishIndexMetrics.new({type: "extract_convert_index_data"}) - build_dir = Pathname.new(Dromedary.config.build_dir).realdirpath - build_dir.mkpath - - xmldir = build_dir + "xml" - xmldir.mkpath - - # Ugh. Need to fix this so it's not so stupid. AnnoyingUtilities - # are too hard-coded. And hence annoying! - - original_data_dir = AnnoyingUtilities.data_dir - AnnoyingUtilities.data_dir = build_dir - - logger.info "Begin extraction from #{zipfile}" - MedInstaller::Extract.new(command_name: "extract").call(zipfile: zipfile, datadir: build_dir) - logger.info "...done" - - logger.info "Begin conversion of data in #{xmldir}" - MedInstaller::Convert.new(command_name: "convert").call(source_dir: build_dir + "xml") - logger.info "...done" - - logger.info "Setting to maintenance mode during indexing" - MedInstaller::Control::MaintenanceModeOn.new(command_name: "maintenance_mode on").call("on") - - logger.info "Copying bib_all.xml to build dir #{build_dir}" - %w[bib_all.xml].each do |f| - path = xmldir + f - FileUtils.copy path, build_dir - end - - logger.info "Begin full index" - MedInstaller::Index::Full.new(command_name: "index full").call(debug: false, existing_hyp_to_bibid: false) - logger.info "...done" - - logger.info "Copying files to live site" - %w[entries.json.gz bib_all.xml hyp_to_bibid.json].each do |f| - path = build_dir + f - if !path.exist? - logger.warn "File #{path} not found, can't be copied to #{original_data_dir}" - end - FileUtils.copy path, original_data_dir - end - - logger.info "New data in place. Making the site live again." - MedInstaller::Control::MaintenanceModeOff.new(command_name: "maintenance_mode off").call("off") - # metrics.log_success - end - end -end diff --git a/lib/med_installer/index.rb b/lib/med_installer/index.rb deleted file mode 100644 index 713b53a..0000000 --- a/lib/med_installer/index.rb +++ /dev/null @@ -1,197 +0,0 @@ -require "hanami/cli" -require "pathname" -require "annoying_utilities" -require "med_installer/logger" -require "med_installer/solr" -require "middle_english_dictionary/collection/bib_set" -require "traject" -require "json" - -module MedInstaller - module Index - class Generic < Hanami::CLI::Command - include MedInstaller::Logger - include AnnoyingUtilities - - def index_dir - AnnoyingUtilities::DROMEDARY_ROOT + "indexer" - end - - def indexing_rules_file - raise "Set in subclass" - end - - def select_writer(debug) - _writer = if debug - index_dir + "writers" + "debug.rb" - else - index_dir + "writers" + "localhost.rb" - end - end - - def core - core = AnnoyingUtilities.solr_core - # Commit with index building can take a looooong time. Set the timeout to something long - core.rawclient.receive_timeout = 200_000 # 200 seconds - core - end - - def index(rulesfile:, datafile:, bibfile:, writer:) - indexer = ::Traject::Indexer.new - indexer.settings do - store "med.data_file", datafile.to_s - store "bibfile", bibfile - end - - indexer.load_config_file rulesfile.to_s - indexer.load_config_file writer.to_s - exitstatus = indexer.process(File.open("/dev/null")) - logger.info "Traject running #{rulesfile} exited with status #{exitstatus}" - end - - def call(debug:) - raise "Solr at #{AnnoyingUtilities.blacklight_solr_url} not up" unless AnnoyingUtilities.solr_core.up? - writer = select_writer(debug) - fields = indexing_rules_file - index(rulesfile: fields, datafile: filename, writer: writer, bibfile: bibfile) - end - - def commit - logger.info "Sending commit" - core.commit - end - - def optimize - logger.info "Optimizing (long!)" - core.optimize - end - end - - # ----- - - class Entries < Generic - desc "Index entries into solr using the traject configuration in indexer/main_indexing_rules" - - option :debug, type: :boolean, default: false, desc: "Write to debug file?" - - def indexing_rules_file - index_dir + "main_indexing_rules.rb" - end - - def call(debug:) - HypToBibID.new(command_name: "hyp_to_bib_id").call - raise "Solr at #{AnnoyingUtilities.blacklight_solr_url} not up" unless AnnoyingUtilities.solr_core.up? - writer = select_writer(debug) - fields = indexing_rules_file - index(rulesfile: fields, datafile: AnnoyingUtilities.entries_path, writer: writer, bibfile: AnnoyingUtilities.bibfile_path) - end - end - - # ------ - # - - class Bib < Generic - desc "Index entries into solr using the traject configuration in indexer/bib_indexing_rules" - option :debug, type: :boolean, default: false, desc: "Write to debug file?" - - def indexing_rules_file - index_dir + "bib_indexing_rules.rb" - end - - def call(debug:) - HypToBibID.new(command_name: "hyp_to_bib_id").call - raise "Solr at #{AnnoyingUtilities.blacklight_solr_url} not up" unless AnnoyingUtilities.solr_core.up? - writer = select_writer(debug) - index(rulesfile: index_dir + "bib_indexing_rules.rb", - datafile: AnnoyingUtilities.bibfile_path, - writer: writer, - bibfile: AnnoyingUtilities.bibfile_path) - commit - optimize - commit - end - end - - #------ - - class Full < Generic - desc "Clear and reload solr, index entries and bib, build autosuggest, and optimize" - option :debug, type: :boolean, default: false, desc: "Write to debug file?" - option :existing_hyp_to_bibid, type: :boolean, default: false, desc: "Don't create new hyp_to_bibid" - option :build_directory, type: :string, required: false, - default: Dromedary::Services[:build_directory], - desc: "The build directory (contains entries.json and xml/)" - def call(debug:, existing_hyp_to_bibid:, build_directory:) - raise "Solr at #{AnnoyingUtilities.blacklight_solr_url} not up" unless AnnoyingUtilities.solr_core.up? - Dromedary::Services.register(:build_directory) { build_directory } - writer = select_writer(debug) - - logger.info "Clearing existing data" - core.clear - - logger.info "Reloading core definition" - core.reload - - HypToBibID.new(command_name: "hyp_to_bib_id").call unless existing_hyp_to_bibid - - logger.info "Setting to maintenance mode during indexing" - MedInstaller::Control::MaintenanceModeOn.new(command_name: "maintenance_mode on").call("on") - - logger.info "##### BEGIN ENTRY/QUOTE INDEXING #####" - index(rulesfile: index_dir + "main_indexing_rules.rb", - datafile: AnnoyingUtilities.entries_path, - writer: writer, - bibfile: AnnoyingUtilities.bibfile_path) - - logger.info "##### BEGIN BIB INDEXING #####" - - index(rulesfile: index_dir + "bib_indexing_rules.rb", - datafile: AnnoyingUtilities.bibfile_path, - writer: writer, - bibfile: AnnoyingUtilities.bibfile_path) - commit - MedInstaller::Solr.rebuild_suggesters(core) - commit - optimize - logger.info "Done" - logger.info "New data in place. Making the site live again." - MedInstaller::Control::MaintenanceModeOff.new(command_name: "maintenance_mode off").call("off") - end - end - - # ----- - - class HypToBibID < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Create the mapping from HYP ids (RID) to bib IDs" - - def bibset(filename) - @bibset ||= MiddleEnglishDictionary::Collection::BibSet.new(filename: filename) - end - - def hyp_to_bibid - return @hyp_to_bibid if @hyp_to_bibid - logger.info "Building hyp_to_bibid mapping" - bibfile = Pathname.new(Dromedary::Services[:build_directory]) + "xml" + "bib_all.xml" - @hyp_to_bibid ||= bibset(bibfile).each_with_object({}) do |bib, acc| - bib.hyps.each do |hyp| - acc[hyp.delete("\\").upcase] = bib.id # TODO: Take out when backslashes removed from HYP ids - end - end - end - - def write_hyp_to_bib_id - logger.info "Creating and writing hyp_to_bibid mapping at #{AnnoyingUtilities.hyp_to_bibid_path}" - File.open(AnnoyingUtilities.hyp_to_bibid_path, "w:utf-8") do |out| - out.puts hyp_to_bibid.to_json - end - end - - def call(command_name: "HypToBibID") - logger.info "Creating hyp_to_bibid.json" - write_hyp_to_bib_id - end - end - end -end diff --git a/lib/med_installer/index_new_data.rb b/lib/med_installer/index_new_data.rb deleted file mode 100644 index 72ab419..0000000 --- a/lib/med_installer/index_new_data.rb +++ /dev/null @@ -1,19 +0,0 @@ -require_relative "index" -require_relative "control" -require_relative "copy_from_build" -require_relative "../dromedary/services" -module MedInstaller - - class IndexNewData < Hanami::CLI::Command - include MedInstaller::Logger - - option :build_directory, - required: false, - default: Dromedary::Services[:build_directory], - desc: "The build directory with entries.json.gz and hyp_to_bibid.json" - - def call(build_directory:) - MedInstaller::Index::Full.new(command_name: "index full").call(debug: false, existing_hyp_to_bibid: false, build_directory: build_directory) - end - end -end diff --git a/lib/med_installer/remote.rb b/lib/med_installer/remote.rb deleted file mode 100644 index 89a9172..0000000 --- a/lib/med_installer/remote.rb +++ /dev/null @@ -1,94 +0,0 @@ -require "hanami/cli" -require "annoying_utilities" -require_relative "logger" - -module MedInstaller - class Remote - extend MedInstaller::Logger - - VALID_TARGETS = %w[testing staging production] - PANIC_PAUSE = 5 - - def self.valid_target?(t) - target = t.downcase - VALID_TARGETS.include? target - end - - def self.validate_target!(t) - if valid_target?(t) - t.downcase - else - raise "Target must be one of [#{VALID_TARGETS.join(", ")}]" - end - end - - def self.remote_exec(target, cmd) - logger.info "Telling #{target} to run #{cmd}" - # full_command = "ssh deployhost exec -v --env=RAILS_ENV:production dromedary-#{target} app ruby #{cmd}" - full_command = %(ssh deployhost exec dromedary-#{target} "#{cmd}") - system full_command - end - - class Deploy < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Deploy to a valid target (testing/staging/production)" - argument :target, required: true, desc: "Which deployment (testing/staging/production)" - argument :branch, default: "master", desc: "Which branch/tag/SHA to deploy" - - def call(target:, branch:) - target = Remote.validate_target!(target) - logger.info "Deploying #{branch} to #{target.upcase}" - sleep(Remote::PANIC_PAUSE) - cmd = "ssh deployhost deploy dromedary-#{target} #{branch}" - logger.info cmd - system cmd - end - end - - class Restart < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Restart the puma server for a valid target" - argument :target, required: true, desc: "Which deployment (testing/staging/production)" - - def call(target:) - target = Remote.validate_target!(target) - logger.info "Restarting puma server for #{target.upcase}" - system "ssh deployhost restart dromedary-#{target}" - end - end - - class Dromedary < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Run a bin/dromedary command on a remote server" - argument :target, required: true, desc: "Which deployment (testing/staging/production)" - argument :command, required: true, desc: "The command to run (e.g., \"solr reload\" IN DOUBLE QUOTES)" - argument :arg1, required: false, default: "" - argument :arg2, required: false, default: "" - argument :arg3, required: false, default: "" - - def call(target:, command:, arg1:, arg2:, arg3:) - target = Remote.validate_target!(target) - sleep(Remote::PANIC_PAUSE) - cmd = %(bin/dromedary #{[command, arg1, arg2, arg3].join(" ").strip}) - Remote.remote_exec(target, cmd) - end - end - - class Exec < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Run an arbitrary command using deploy exec" - argument :target, required: true, desc: "Which deployment (testing/staging/production)" - argument :command, required: true, desc: "The command to run (e.g., \"curl http://localhost...\" IN DOUBLE QUOTES)" - - def call(target:, command:) - target = Remote.validate_target!(target) - sleep(Remote::PANIC_PAUSE) - Remote.remote_exec(target, command) - end - end - end -end diff --git a/lib/med_installer/solr.rb b/lib/med_installer/solr.rb deleted file mode 100644 index fef31c3..0000000 --- a/lib/med_installer/solr.rb +++ /dev/null @@ -1,276 +0,0 @@ -require "middle_english_dictionary" -require "hanami/cli" -require "annoying_utilities" -require "simple_solr_client" - -require_relative "logger" - -require "my_simple_solr_client" - -Zip.on_exists_proc = true - -# update 2019_updates u, slip_rights sr -# set sr.attr = u.rights_current_attr -# where u.slip_rights_attr <> sr.attr and u.full_id = sr.nid; - -module MedInstaller - class Solr - extend MedInstaller::Logger - URL = "http://mirrors.gigenet.com/apache/lucene/solr/6.6.3/solr-6.6.3.tgz" - DIR_EXTRACTED_FROM_SOLR_TARGZ = "solr-6.6.3" # make this better! - - DROMEDARY_ROOT = AnnoyingUtilities::DROMEDARY_ROOT - MED_CONFIG = DROMEDARY_ROOT + "solr" + "med" - SOLR_LIBS = DROMEDARY_ROOT + "solr" + "lib" - DOT_SOLR = AnnoyingUtilities::DOT_SOLR - DEFAULT_SOLR = AnnoyingUtilities::DEFAULT_SOLR - - def self.get_port_with_logging(rails_env) - p = AnnoyingUtilities.solr_port - if p - logger.info "Got port #{p} from the solr url in blacklight_config.yml" - p - else - logger.warn "Didn't find a port in the url string in blacklight.yml; using 9639" - "9639" - end - end - - def self.rebuild_suggesters(core, env = nil) - envenv = ENV["RAILS_ENV"] - env ||= if envenv.nil? || envenv.empty? - "development" - else - envenv - end - logger.info "Recreating suggest indexes for #{env} environment" - autocomplete = AnnoyingUtilities.load_config_file("autocomplete.yml")[env] - autocomplete.keys.each do |key| - suggester_path = autocomplete[key]["solr_endpoint"] - logger.info " Recreate suggester for #{suggester_path}" - # _resp = core.get "config/#{suggester_path}", {"suggest.build" => "true"} - connection = MySimpleSolrClient::Client.new(Dromedary::Services[:solr_embedded_auth_url]) - resp = connection.solr_connection.get "#{suggester_path}", {"suggest.build" => "true"} - end - end - - class Commit < Hanami::CLI::Command - include MedInstaller::Logger - desc "Force solr to commit" - - def call(cmd) - core = AnnoyingUtilities.solr_core - - unless core.up? - logger.error "Solr core at #{core.url} did not respond (not up?)" - exit(1) - end - - core.commit - logger.info "Core at '#{core.url}' sent commit" - end - end - - class Optimize < Hanami::CLI::Command - include MedInstaller::Logger - desc "Optimize solr index" - - def call(cmd) - core = AnnoyingUtilities.solr_core - - unless core.up? - logger.error "Solr core at #{core.url} did not respond (not up?)" - exit(1) - end - - logger.info "Begining optimization (can be *long*)" - core.optimize - logger.info "Core at '#{core.url}' optimized" - end - end - - class Reload < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Tell solr to reload the solr config without restarting" - - def call(cmd) - core = AnnoyingUtilities.solr_core - - unless core.up? - logger.error "Solr core at #{core.url} did not respond (not up?)" - exit(1) - end - - core.reload - logger.info "Core at '#{core.url}' reloaded" - end - end - - class RebuildSuggesters < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Tell solr to rebuild all the suggester indexes" - - def call(cmd) - core = AnnoyingUtilities.solr_core - Solr.rebuild_suggesters(core) - end - end - - class Empty < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Delete all documents in the solr" - - def call(cmd) - core = AnnoyingUtilities.solr_core - unless core.up? - logger.error "Solr core at #{core.url} did not respond (not up?)" - exit(1) - end - core.clear.commit - logger.info "Solr core at #{core.url} emptied out" - end - end - - class Install < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Download and install solr to the given directory" - - option :installdir, default: AnnoyingUtilities::DROMEDARY_ROOT.parent, desc: "The install directory (default: next to dromedary)" - - def call(installdir:) - installpath = Pathname(installdir).realdirpath - solrpath = installpath + DIR_EXTRACTED_FROM_SOLR_TARGZ - lnpath = installpath + "solr" - solr_solr_dir = lnpath + "server" + "solr" - _solr_config_dir = solr_solr_dir + "med" - _solr_lib_dir = solr_solr_dir + "lib" - - logger.info "Download/extract from #{URL}" - logger.info "Installing in directory #{installpath}" - status = system(%(curl '#{URL}' | tar -C '#{installpath}' -x -z -f -)) - - raise "Something went wrong with download / extract: #{status}" unless status - - logger.info "Making a symlink so we can use #{lnpath} instead of #{solrpath}" - lncmd = "rm -f '#{lnpath}'; ln -s '#{solrpath}' '#{lnpath}'" - status = system lncmd - raise "Trouble symlinking #{solrpath} to #{lnpath}" unless status - - logger.info "Storing path to solr directory in dromedary/.solr" - File.open(DOT_SOLR, "w:utf-8") do |out| - out.puts lnpath.to_s - end - Link.new(command_name: "solr link").call("solr link") - rescue => err - logger.error err.message - logger.error err.backtrace - logger.error "Exiting" - exit(1) - end - end - - class Link < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Link in the MED solr configurations to the solr in .solr" - - def call(cmd) - solr_root = AnnoyingUtilities.solr_root - solr_solr_dir = solr_root + "server" + "solr" - solr_config_dir = solr_solr_dir + "med" - solr_lib_dir = Pathname.new(AnnoyingUtilities.data_dir) + "lib" - - logger.info "Linking dromedary solr config stuff into the data dir" - logger.info "Found solr directory #{solr_root}" - logger.info "Linking #{Solr::MED_CONFIG} into #{solr_config_dir}" - status = system "rm -f '#{solr_config_dir}'; ln -s '#{Solr::MED_CONFIG}' '#{solr_config_dir}'" - raise "Trouble linking #{Solr::MED_CONFIG} into the right place in solr" unless status - - logger.info "Linking in #{Solr::SOLR_LIBS}" - status = system "rm -f '#{solr_lib_dir}'; ln -s '#{Solr::SOLR_LIBS}' '#{solr_lib_dir}'" - raise "Trouble linking #{Solr::SOLR_LIBS}" unless status - logger.info "Done" - rescue => err - logger.error err.message - logger.error err.backtrace - logger.error "Exiting" - exit(1) - end - end - - class Start < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Start the solr referenced in .solr" - argument :rails_env, required: false, default: "development", desc: "The rails environment" - - def solr_bin - AnnoyingUtilities.solr_root + "bin" + "solr" - end - - def call(rails_env:) - port = Solr.get_port_with_logging(rails_env) - portarg = "-p #{port}" - command = "#{solr_bin} restart #{portarg} -Dsolr.solr.home=#{AnnoyingUtilities.solr_dir} -Ddromedary.data_dir=\"#{AnnoyingUtilities.data_dir}\"" - logger.info "Starting solr with command:\n #{command}" - system command - end - end - - class Stop < Start - argument :rails_env, default: "development", required: false, desc: "The rails environment" - - def call(rails_env:) - port = Solr.get_port_with_logging(rails_env) - portarg = "-p #{port}" - system "#{solr_bin} stop #{portarg}" - end - end - - class Up < Hanami::CLI::Command - include MedInstaller::Logger - - desc "Check to see if solr is up" - - def call(cmd) - core = AnnoyingUtilities.solr_core - - if core.up? - logger.info "Solr at #{core.url} appears to be up and running" - else - logger.error "Solr core at #{core.url} did not respond (not up?)" - end - end - end - - class Shell < Hanami::CLI::Command - desc "Get a shell connected to solr, optionally with collections" - - option :entries, required: false, desc: "Path to the entries.json.gz file (exposed as `entry_set`)" - option :bibs, required: false, desc: "Path to the bib_all.xml (exposed as `bib_set`)" - - def call(entries: nil, bibs: nil, **kw) - Up.new(command_name: "shell").call("shell") - core = AnnoyingUtilities.solr_core - - entry_set = if entries - settings = { - "med.data_file" => entries - } - MedInstaller::EntryJsonReader.new(settings) - end - - bib_set = if bibs - MiddleEnglishDictionary::Collection::BibSet.new(filename: bibs) - end - - require "pry" - binding.pry # standard:disable Lint/Debugger - end - end - end -end