From 51b6ed551d70b2322074e9d1ff21950f5978636c Mon Sep 17 00:00:00 2001 From: "pzupan@gmail.com" Date: Thu, 19 Dec 2019 16:05:30 -0800 Subject: [PATCH 1/5] allow for readme override --- ...c_extension_contents_at_versions_worker.rb | 37 ++++++++++++++++++- ...le_github_extension_version_config_spec.rb | 2 +- .../find_or_create_release_asset_spec.rb | 2 - ...ension_contents_at_versions_worker_spec.rb | 29 ++++++++++++--- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/app/workers/sync_extension_contents_at_versions_worker.rb b/app/workers/sync_extension_contents_at_versions_worker.rb index 7a39d05a..a7e10c96 100755 --- a/app/workers/sync_extension_contents_at_versions_worker.rb +++ b/app/workers/sync_extension_contents_at_versions_worker.rb @@ -49,6 +49,7 @@ def sync_extension_version(release_info) set_commit_count(version) scan_files(version) sync_release_info(version, release_info) + check_for_overrides(version) PersistAssets.call(version: version) update_annotations(version) version.save @@ -65,14 +66,14 @@ def semver? begin tag = SemverNormalizer.call(@tag) - puts "Normalized Tag: #{tag}" + # puts "Normalized Tag: #{tag}" Semverse::Version.new(tag) return true rescue Semverse::InvalidVersionFormat => error unless @errored_tags.include?(@tag) @errored_tags << @tag compilation_error = [@extension.compilation_error, error.message] - @extension.update_column(:compilation_error, compilation_error.reject(&:empty?).join('; ')) + @extension.update_column(:compilation_error, compilation_error.compact.join('; ')) message = "#{@extension.lowercase_name} release is invalid: #{error.message}" logger.info message end @@ -100,6 +101,38 @@ def fetch_readme end end + def check_for_overrides(version) + return if (overrides = version.config['overrides']).blank? + if (readme_url = overrides["readme_url"]).present? + override_readme(version, readme_url) + end + end + + def override_readme(version, readme_url) + begin + url = URI.parse(readme_url) + rescue URI::Error => error + logger.info "URI error: #{readme_url} - #{error.message}" + return + end + # get file contents + begin + file = url.open + rescue OpenURI::HTTPError => error + status = error.io.status + logger.info "****** file read error: #{status[0]} - #{status[1]}" + return + end + readme_ext = File.extname(url.path).gsub(".", "") + readme = file.read + readme = readme.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''}) + version.update( + readme: readme, + readme_extension: readme_ext + ) + logger.info "OVERRIDE README: #{url.path}" + end + def extract_readme_file_extension(filename) if match = filename.match(/\.[a-zA-Z0-9]+$/) match[0].gsub(".", "") diff --git a/spec/interactors/compile_github_extension_version_config_spec.rb b/spec/interactors/compile_github_extension_version_config_spec.rb index 7a53f090..fbb12023 100755 --- a/spec/interactors/compile_github_extension_version_config_spec.rb +++ b/spec/interactors/compile_github_extension_version_config_spec.rb @@ -13,7 +13,7 @@ "asset_filename"=>"\#{repo}-\#{version}-linux-x86_64.tar.gz"}], "labels"=> {"example"=> "custom"}, "annotations"=> {"io.sensu.bonsai.test"=> "test value"}, - "description"=> "test asset" + "description"=>"test asset" } } let(:extension) { create :extension, extension_versions_count: 0, github_url: "https://github.com/owner/#{repo_name}" } let(:version) { create :extension_version, extension: extension, config: config, version: version_name } diff --git a/spec/interactors/find_or_create_release_asset_spec.rb b/spec/interactors/find_or_create_release_asset_spec.rb index 4ac4d4bf..f213e33f 100755 --- a/spec/interactors/find_or_create_release_asset_spec.rb +++ b/spec/interactors/find_or_create_release_asset_spec.rb @@ -2,8 +2,6 @@ describe FindOrCreateReleaseAsset do - - describe ".call" do context "when valid version and build" do diff --git a/spec/workers/sync_extension_contents_at_versions_worker_spec.rb b/spec/workers/sync_extension_contents_at_versions_worker_spec.rb index 3805a872..536d2fc8 100755 --- a/spec/workers/sync_extension_contents_at_versions_worker_spec.rb +++ b/spec/workers/sync_extension_contents_at_versions_worker_spec.rb @@ -28,11 +28,11 @@ @s3_bucket = s3.bucket(ENV['AWS_S3_ASSETS_BUCKET']) # trap incorrect ENV variables (Travis) or no network connection - begin - stub_aws unless @s3_bucket.exists? - rescue + #begin + # stub_aws unless @s3_bucket.exists? + #rescue stub_aws - end + #end end @@ -41,6 +41,8 @@ def stub_aws Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :exists?).and_return(true) Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :public_url).and_return('https://s3.us-west-2.amazonaws.com/bucket/example.com') Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :last_modified).and_return(DateTime.now.to_s(:db)) + Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :delete).and_return(true) + Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :put).and_return(true) end describe 'tag checking' do @@ -92,7 +94,7 @@ def stub_aws it 'puts annotations in annotations field' do config_hash['annotations'] = { suggested_asset_url: '/suggested/asset', - suggested_asset_message: 'Suggested Asset Messaage' + suggested_asset_message: 'Suggested Asset Message' } version.update_column(:config, config_hash) subject.perform(extension.id, tags) @@ -103,4 +105,21 @@ def stub_aws end + describe 'overrides' do + let(:config_hash) { version.config } + let(:tags) { [version.version] } + + it 'loads an alternate readme file' do + config_hash["overrides"] = { + "readme_url"=>"https://github.com/sensu/bonsai/blob/master/README.md" + } + version.update_column(:config, config_hash) + subject.perform(extension.id, tags) + version.reload + expect(version.readme).to include('bonsai.sensu.io') + end + + end + + end From 186dc34c12b4677a8c3222a15263766a662f91a4 Mon Sep 17 00:00:00 2001 From: "pzupan@gmail.com" Date: Fri, 20 Dec 2019 12:48:11 -0800 Subject: [PATCH 2/5] fix bug in sync worker for override --- app/workers/sync_extension_contents_at_versions_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/sync_extension_contents_at_versions_worker.rb b/app/workers/sync_extension_contents_at_versions_worker.rb index a7e10c96..972bae0a 100755 --- a/app/workers/sync_extension_contents_at_versions_worker.rb +++ b/app/workers/sync_extension_contents_at_versions_worker.rb @@ -103,7 +103,7 @@ def fetch_readme def check_for_overrides(version) return if (overrides = version.config['overrides']).blank? - if (readme_url = overrides["readme_url"]).present? + if (readme_url = overrides[:readme_url]).present? override_readme(version, readme_url) end end From bce83426331087ed7d58c06eaa9de1910f2071fa Mon Sep 17 00:00:00 2001 From: "pzupan@gmail.com" Date: Fri, 20 Dec 2019 13:10:20 -0800 Subject: [PATCH 3/5] fix bug 2 in sync worker for override --- app/workers/sync_extension_contents_at_versions_worker.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/workers/sync_extension_contents_at_versions_worker.rb b/app/workers/sync_extension_contents_at_versions_worker.rb index 972bae0a..84d97bb4 100755 --- a/app/workers/sync_extension_contents_at_versions_worker.rb +++ b/app/workers/sync_extension_contents_at_versions_worker.rb @@ -102,9 +102,10 @@ def fetch_readme end def check_for_overrides(version) - return if (overrides = version.config['overrides']).blank? - if (readme_url = overrides[:readme_url]).present? - override_readme(version, readme_url) + return if version.blank? || version.config["overrides"].nil? + overrides = version.config["overrides"] + unless overrides["readme_url"].nil? + override_readme(version, overrides["readme_url"]) end end From 87fceceb35bc1486375debf370f53dc4d7f7d401 Mon Sep 17 00:00:00 2001 From: "pzupan@gmail.com" Date: Fri, 20 Dec 2019 13:54:41 -0800 Subject: [PATCH 4/5] fix datatypes in override --- .../sync_extension_contents_at_versions_worker.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/workers/sync_extension_contents_at_versions_worker.rb b/app/workers/sync_extension_contents_at_versions_worker.rb index 84d97bb4..680790f3 100755 --- a/app/workers/sync_extension_contents_at_versions_worker.rb +++ b/app/workers/sync_extension_contents_at_versions_worker.rb @@ -13,7 +13,7 @@ def perform(extension_id, tags, compatible_platforms = [], release_infos_by_tag @extension = Extension.find_by(id: extension_id) raise RuntimeError.new("#{I18n.t('nouns.extension')} ID: #{extension_id.inspect} not found.") unless @extension - puts "PERFORMING #{@extension.name}: #{tags.join(', ')}" + #puts "PERFORMING #{@extension.name}: #{tags.join(', ')}" logger.info("PERFORMING: #{@extension.inspect}, #{tags.inspect}, #{compatible_platforms.inspect}") @errored_tags = [] @@ -103,8 +103,8 @@ def fetch_readme def check_for_overrides(version) return if version.blank? || version.config["overrides"].nil? - overrides = version.config["overrides"] - unless overrides["readme_url"].nil? + overrides = version.config["overrides"][0] + if overrides["readme_url"].present? override_readme(version, overrides["readme_url"]) end end @@ -124,12 +124,13 @@ def override_readme(version, readme_url) logger.info "****** file read error: #{status[0]} - #{status[1]}" return end - readme_ext = File.extname(url.path).gsub(".", "") + #readme_ext = File.extname(url.path).gsub(".", "") readme = file.read - readme = readme.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''}) + #readme = readme.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''}) + version.update( readme: readme, - readme_extension: readme_ext + readme_extension: 'html' ) logger.info "OVERRIDE README: #{url.path}" end From f598aefd92b72dbeb3bd2ea338987f65279683a1 Mon Sep 17 00:00:00 2001 From: "pzupan@gmail.com" Date: Fri, 20 Dec 2019 15:13:56 -0800 Subject: [PATCH 5/5] capture invalid markdown in override --- app/helpers/extension_versions_helper.rb | 3 +- app/models/extension_version.rb | 2 + ...c_extension_contents_at_versions_worker.rb | 51 +++++++++++++++---- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/app/helpers/extension_versions_helper.rb b/app/helpers/extension_versions_helper.rb index d96b0990..b7e1fe3e 100755 --- a/app/helpers/extension_versions_helper.rb +++ b/app/helpers/extension_versions_helper.rb @@ -64,8 +64,9 @@ def extension_atom_content(extension_version) # def render_document(content, extension, repo_loc = "", version = "", hard_wrap: false) document = begin - if %w(md mdown markdown).include?(extension.downcase) + if ExtensionVersion::MARKDOWN_EXTENSIONS.include?(extension.downcase) filter = HTML::Pipeline.new [ + HTML::Pipeline::PlainTextInputFilter, HTML::Pipeline::MarkdownFilter, HTML::Pipeline::AutolinkFilter, HTML::Pipeline::MentionFilter, diff --git a/app/models/extension_version.rb b/app/models/extension_version.rb index 28d5c888..8e8766be 100755 --- a/app/models/extension_version.rb +++ b/app/models/extension_version.rb @@ -1,5 +1,7 @@ class ExtensionVersion < ApplicationRecord include SeriousErrors + + MARKDOWN_EXTENSIONS = %w(markdown mdown mkdn mkd md) # Associations # -------------------- diff --git a/app/workers/sync_extension_contents_at_versions_worker.rb b/app/workers/sync_extension_contents_at_versions_worker.rb index 680790f3..927db2d1 100755 --- a/app/workers/sync_extension_contents_at_versions_worker.rb +++ b/app/workers/sync_extension_contents_at_versions_worker.rb @@ -1,4 +1,5 @@ class SyncExtensionContentsAtVersionsWorker < ApplicationWorker + include MarkdownHelper def logger if Rails.env.production? @@ -110,29 +111,61 @@ def check_for_overrides(version) end def override_readme(version, readme_url) + + message = [] + begin url = URI.parse(readme_url) rescue URI::Error => error logger.info "URI error: #{readme_url} - #{error.message}" return end + + readme_ext = File.extname(url.path).gsub(".", "") + unless ExtensionVersion::MARKDOWN_EXTENSIONS.include?(readme_ext) + message << "#{version.version} override readme_url is not a valid markdown file." + end + # get file contents begin file = url.open rescue OpenURI::HTTPError => error status = error.io.status - logger.info "****** file read error: #{status[0]} - #{status[1]}" + message << "#{version.version} #{url.path} file read error: #{status[0]} - #{status[1]}" + logger.info message.compact.join('; ') + compilation_error = [version.compilation_error] + message + version.update_column(:compilation_error, compilation_error.compact.join('; ')) return end - #readme_ext = File.extname(url.path).gsub(".", "") - readme = file.read - #readme = readme.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''}) - version.update( - readme: readme, - readme_extension: 'html' - ) - logger.info "OVERRIDE README: #{url.path}" + readme = file.read + + if readme.include?('!DOCTYPE html') + message << "#{version.version} override readme is not valid markdown." + end + + begin + filter = HTML::Pipeline.new [ + HTML::Pipeline::MarkdownFilter, + ], {gfm: true} + filter.call(readme) + rescue + message << "#{version.version} override readme is not valid markdown." + end + + if message.present? + compilation_error = [version.compilation_error] + message + version.update_column(:compilation_error, compilation_error.compact.join('; ')) + else + + readme = readme.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''}) + + version.update( + readme: readme, + readme_extension: readme_ext + ) + logger.info "OVERRIDE README: #{url.path}" + end end def extract_readme_file_extension(filename)