From 0e739795c2b5ec1cfa35aa9cb8d9562bbd9d36b0 Mon Sep 17 00:00:00 2001 From: Ben Abrams Date: Fri, 7 Oct 2016 20:26:54 -0700 Subject: [PATCH] first pass at refactor logic into lib added .rubocop.yml updated rest-client removed unsafe eval --- .gitignore | 3 + CHANGELOG.md | 1 + aws-cleaner.gemspec | 6 +- bin/aws_cleaner.rb | 207 ++++----------------------------- lib/aws_cleaner/aws_cleaner.rb | 199 +++++++++++++++++++++++++++++++ rubocop.yml | 99 ++++++++++++++++ 6 files changed, 329 insertions(+), 186 deletions(-) create mode 100644 lib/aws_cleaner/aws_cleaner.rb create mode 100644 rubocop.yml diff --git a/.gitignore b/.gitignore index 1ccd44b..e4147ee 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,6 @@ Gemfile.lock # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: .rvmrc + +### aws_cleaner config ### +config.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 21a18c8..5fc63cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ### Unreleased +- refactor logic into a library ### 2.0.1 - actually add `slack-poster` dependency diff --git a/aws-cleaner.gemspec b/aws-cleaner.gemspec index 8523932..1b116a3 100644 --- a/aws-cleaner.gemspec +++ b/aws-cleaner.gemspec @@ -6,14 +6,16 @@ Gem::Specification.new do |s| s.authors = ['Eric Heydrick'] s.email = 'eheydrick@gmail.com' s.executables = ['aws_cleaner.rb'] - s.files = ['bin/aws_cleaner.rb'] + s.files = Dir.glob("{bin,lib}/**/*.rb") s.homepage = 'https://github.com/eheydrick/aws-cleaner' s.license = 'MIT' + s.add_development_dependency 'rubocop', '~> 0.43.0' + s.add_runtime_dependency 'aws-sdk-core', '~> 2.0' s.add_runtime_dependency 'chef-api', '~> 0.5' s.add_runtime_dependency 'hipchat', '~> 1.5' - s.add_runtime_dependency 'rest-client', '~> 1.8' + s.add_runtime_dependency 'rest-client', '~> 2 ' s.add_runtime_dependency 'slack-poster', '~> 2.2' s.add_runtime_dependency 'trollop', '~> 2.1' end diff --git a/bin/aws_cleaner.rb b/bin/aws_cleaner.rb index 07b1f12..9bdaed6 100755 --- a/bin/aws_cleaner.rb +++ b/bin/aws_cleaner.rb @@ -8,6 +8,7 @@ # Licensed under The MIT License # +# ensure gems are present begin require 'json' require 'yaml' @@ -21,6 +22,9 @@ raise "Missing gems: #{e}" end +# require our class +require_relative '../lib/aws_cleaner/aws_cleaner.rb' + def config(file) YAML.load(File.read(file)) rescue StandardError => e @@ -28,185 +32,21 @@ def config(file) end # get options -opts = Trollop::options do +opts = Trollop.options do opt :config, 'Path to config file', type: :string, default: 'config.yml' end @config = config(opts[:config]) -@sqs = Aws::SQS::Client.new(@config[:aws]) - -@chef = ChefAPI::Connection.new( - endpoint: @config[:chef][:url], - client: @config[:chef][:client], - key: @config[:chef][:key] -) - -# delete the message from SQS -def delete_message(id) - delete = @sqs.delete_message( - queue_url: @config[:sqs][:queue], - receipt_handle: id - ) - delete ? true : false -end - -# return the body of the SQS message in JSON -def parse(body) - JSON.parse(body) -rescue JSON::ParserError - return false -end - -# return the instance_id of the terminated instance -def process_message(message_body) - return false if message_body['detail']['instance-id'].nil? && - message_body['detail']['state'] != 'terminated' - - instance_id = message_body['detail']['instance-id'] - instance_id -end - -# call the Chef API to get the node name of the instance -def get_chef_node_name(instance_id) - results = @chef.search.query(:node, "ec2_instance_id:#{instance_id} OR chef_provisioning_reference_server_id:#{instance_id}") - if results.rows.size > 0 - return results.rows.first['name'] - else - return false - end -end - -# call the Chef API to get the FQDN of the instance -def get_chef_fqdn(instance_id) - results = @chef.search.query(:node, "ec2_instance_id:#{instance_id} OR chef_provisioning_reference_server_id:#{instance_id}") - if results.rows.size > 0 - return results.rows.first['automatic']['fqdn'] - else - return false - end -end - -# check if the node exists in Sensu -def in_sensu?(node_name) - begin - RestClient::Request.execute(url: "#{@config[:sensu][:url]}/clients/#{node_name}", method: :get, timeout: 5, open_timeout: 5) - rescue RestClient::ResourceNotFound - return false - rescue => e - puts "Sensu request failed: #{e}" - return false - else - return true - end -end - -# call the Sensu API to remove the node -def remove_from_sensu(node_name) - response = RestClient::Request.execute(url: "#{@config[:sensu][:url]}/clients/#{node_name}", method: :delete, timeout: 5, open_timeout: 5) - case response.code - when 202 - notify_chat('Removed ' + node_name + ' from Sensu') - return true - else - notify_chat('Failed to remove ' + node_name + ' from Sensu') - return false - end -end - -# call the Chef API to remove the node -def remove_from_chef(node_name) - begin - client = @chef.clients.fetch(node_name) - client.destroy - node = @chef.nodes.fetch(node_name) - node.destroy - rescue => e - puts "Failed to remove chef node: #{e}" - else - notify_chat('Removed ' + node_name + ' from Chef') - end -end - -# notify hipchat -def notify_hipchat(msg) - hipchat = HipChat::Client.new( - @config[:hipchat][:api_token], - api_version: 'v2' - ) - room = @config[:hipchat][:room] - hipchat[room].send('AWS Cleaner', msg) -end - -# notify slack -def notify_slack(msg) - slack = Slack::Poster.new(@config[:slack][:webhook_url]) - slack.channel = @config[:slack][:channel] - slack.username = @config[:slack][:username] ||= 'aws-cleaner' - slack.icon_emoji = @config[:slack][:icon_emoji] ||= nil - slack.send_message(msg) -end - -# generic chat notification method -def notify_chat(msg) - if @config[:hipchat][:enable] - notify_hipchat(msg) - elsif @config[:slack][:enable] - notify_slack(msg) - end -end - -# generate the URL for the webhook -def generate_template(item, template_variable_method, template_variable_argument, template_variable) - begin - replacement = send(template_variable_method, eval(template_variable_argument)) - item.gsub!(/{#{template_variable}}/, replacement) - rescue Exception => e - puts "Error generating template: #{e.message}" - return false - else - item - end -end +# @sqs = Aws::SQS::Client.new(@config[:aws]) +@sqs_client = AwsCleaner::SQS.client(@config) -# call an HTTP endpoint -def fire_webhook(config) - # generate templated URL - if config[:template_variables] && config[:url] =~ /\{\S+\}/ - url = generate_template( - config[:url], - config[:template_variables][:method], - config[:template_variables][:argument], - config[:template_variables][:variable] - ) - return false unless url - else - url = config[:url] - end - - hook = { method: config[:method].to_sym, url: url } - r = RestClient::Request.execute(hook) - if r.code != 200 - return false - else - # notify chat when webhook is successful - if config[:chat][:enable] - msg = generate_template( - config[:chat][:message], - config[:chat][:method], - config[:chat][:argument], - config[:chat][:variable] - ) - notify_chat(msg) - end - return true - end -end +@chef_client = AwsCleaner::Chef.client(@config) # main loop loop do # get messages from SQS - messages = @sqs.receive_message( + messages = @sqs_client.receive_message( queue_url: @config[:sqs][:queue], max_number_of_messages: 10, visibility_timeout: 3 @@ -216,53 +56,52 @@ def fire_webhook(config) messages.each_with_index do |message, index| puts "Looking at message number #{index}" - body = parse(message.body) + body = AwsCleaner.new.parse(message.body) id = message.receipt_handle unless body - delete_message(id) + AwsCleaner.new.delete_message(id, @config) next end - @instance_id = process_message(body) + @instance_id = AwsCleaner.new.process_message(body) if @instance_id if @config[:webhooks] - @config[:webhooks].each do |hook, config| - if fire_webhook(config) + @config[:webhooks].each do |hook, hook_config| + if AwsCleaner::Webhooks::fire_webhook(hook_config, @config, @instance_id) puts "Successfully ran webhook #{hook}" else puts "Failed to run webhook #{hook}" end end - delete_message(id) + AwsCleaner.new.delete_message(id, @config) end - chef_node = get_chef_node_name(@instance_id) + chef_node = AwsCleaner::Chef.get_chef_node_name(@instance_id, @config) if chef_node - if remove_from_chef(chef_node) + if AwsCleaner::Chef.remove_from_chef(chef_node, @chef_client, @config) puts "Removed #{chef_node} from Chef" - delete_message(id) + AwsCleaner.new.delete_message(id, @config) end else puts "Instance #{@instance_id} does not exist in Chef, deleting message" - delete_message(id) + AwsCleaner.new.delete_message(id, @config) end - if in_sensu?(chef_node) - if remove_from_sensu(chef_node) + if AwsCleaner::Sensu.in_sensu?(chef_node, @config) + if AwsCleaner::Sensu.remove_from_sensu(chef_node, @config) puts "Removed #{chef_node} from Sensu" - delete_message(id) else puts "Instance #{@instance_id} does not exist in Sensu, deleting message" - delete_message(id) end + AwsCleaner.new.delete_message(id, @config) end else puts 'Message not relevant, deleting' - delete_message(id) + AwsCleaner.new.delete_message(id, @config) end end diff --git a/lib/aws_cleaner/aws_cleaner.rb b/lib/aws_cleaner/aws_cleaner.rb new file mode 100644 index 0000000..dff5f38 --- /dev/null +++ b/lib/aws_cleaner/aws_cleaner.rb @@ -0,0 +1,199 @@ +# main aws_cleaner lib +class AwsCleaner + # SQS related stuff + module SQS + # sqs connection + def self.client(config) + Aws::SQS::Client.new(config[:aws]) + end + end + + # delete the message from SQS + def delete_message(id, config) + delete = AwsCleaner::SQS.client(config).delete_message( + queue_url: config[:sqs][:queue], + receipt_handle: id + ) + delete ? true : false + end + + module Chef + # chef connection + def self.client(config) + ChefAPI::Connection.new( + endpoint: config[:chef][:url], + client: config[:chef][:client], + key: config[:chef][:key] + ) + end + + # call the Chef API to get the node name of the instance + def self.get_chef_node_name(instance_id, config) + chef = client(config) + results = chef.search.query(:node, "ec2_instance_id:#{instance_id} OR chef_provisioning_reference_server_id:#{instance_id}") + return false if results.rows.empty? + results.rows.first['name'] + end + + # call the Chef API to get the FQDN of the instance + def self.get_chef_fqdn(instance_id, config) + chef = client(config) + results = chef.search.query(:node, "ec2_instance_id:#{instance_id} OR chef_provisioning_reference_server_id:#{instance_id}") + return false if results.rows.empty? + results.rows.first['automatic']['fqdn'] + end + + # call the Chef API to remove the node + def self.remove_from_chef(node_name, chef, config) + client = chef.clients.fetch(node_name) + client.destroy + node = chef.nodes.fetch(node_name) + node.destroy + rescue => e + puts "Failed to remove chef node: #{e}" + else + # puts "Removed #{node_name} from chef" + AwsCleaner::Notify.notify_chat('Removed ' + node_name + ' from Chef', config) + end + end + + module Sensu + # check if the node exists in Sensu + def self.in_sensu?(node_name, config) + RestClient::Request.execute( + url: "#{config[:sensu][:url]}/clients/#{node_name}", + method: :get, + timeout: 5, + open_timeout: 5 + ) + rescue RestClient::ResourceNotFound + return false + rescue => e + puts "Sensu request failed: #{e}" + return false + else + return true + end + + # call the Sensu API to remove the node + def self.remove_from_sensu(node_name, config) + response = RestClient::Request.execute( + url: "#{config[:sensu][:url]}/clients/#{node_name}", + method: :delete, + timeout: 5, + open_timeout: 5 + ) + case response.code + when 202 + AwsCleaner::Notify.notify_chat('Removed ' + node_name + ' from Sensu', config) + return true + else + AwsCleaner::Notify.notify_chat('Failed to remove ' + node_name + ' from Sensu', config) + return false + end + end + end + + # return the body of the SQS message in JSON + def parse(body) + JSON.parse(body) + rescue JSON::ParserError + return false + end + + # return the instance_id of the terminated instance + def process_message(message_body) + return false if message_body['detail']['instance-id'].nil? && + message_body['detail']['state'] != 'terminated' + + instance_id = message_body['detail']['instance-id'] + instance_id + end + + module Notify + # notify hipchat + def self.notify_hipchat(msg, config) + hipchat = HipChat::Client.new( + config[:hipchat][:api_token], + api_version: 'v2' + ) + room = config[:hipchat][:room] + hipchat[room].send('AWS Cleaner', msg) + end + + # notify slack + def self.notify_slack(msg, config) + slack = Slack::Poster.new(config[:slack][:webhook_url]) + slack.channel = config[:slack][:channel] + slack.username = config[:slack][:username] ||= 'aws-cleaner' + slack.icon_emoji = config[:slack][:icon_emoji] ||= nil + slack.send_message(msg) + end + + # generic chat notification method + def self.notify_chat(msg, config) + if config[:hipchat][:enable] + notify_hipchat(msg, config) + elsif config[:slack][:enable] + notify_slack(msg, config) + end + end + end + + module Webhooks + # generate the URL for the webhook + def self.generate_template(item, template_variable_method, template_variable_argument, template_variable, config, instance_id) + if template_variable_method == 'get_chef_fqdn' + replacement = AwsCleaner::Chef.get_chef_fqdn(instance_id, config) + elsif template_variable_method == 'get_chef_node_name' + replacement = AwsCleaner::Chef.get_chef_node_name(instance_id, config) + else + raise 'Unknown templating method' + end + item.gsub!(/{#{template_variable}}/, replacement) + rescue StandardError => e + puts "Error generating template: #{e.message}" + return false + else + item + end + + # call an HTTP endpoint + def self.fire_webhook(hook_config, config, instance_id) + # generate templated URL + if hook_config[:template_variables] && hook_config[:url] =~ /\{\S+\}/ + url = AwsCleaner::Webhooks.generate_template( + hook_config[:url], + hook_config[:template_variables][:method], + hook_config[:template_variables][:argument], + hook_config[:template_variables][:variable], + config, + instance_id + ) + return false unless url + else + url = hook_config[:url] + end + + hook = { method: hook_config[:method].to_sym, url: url } + r = RestClient::Request.execute(hook) + if r.code != 200 + return false + else + # notify chat when webhook is successful + if hook_config[:chat][:enable] + msg = AwsCleaner::Webhooks.generate_template( + hook_config[:chat][:message], + hook_config[:chat][:method], + hook_config[:chat][:argument], + hook_config[:chat][:variable], + config, + instance_id + ) + AwsCleaner::Notify.notify_chat(msg, config) + end + return true + end + end + end +end diff --git a/rubocop.yml b/rubocop.yml new file mode 100644 index 0000000..2c3b757 --- /dev/null +++ b/rubocop.yml @@ -0,0 +1,99 @@ +# Rubocop, we're buddies and all, but we're going to have to disagree on the following - + +# Allow compact class definitions +Style/ClassAndModuleChildren: + Enabled: false + +# Allow if statements! +Style/GuardClause: + Enabled: false + +# Allow more complex ruby methods +Metrics/AbcSize: + Max: 100 + +# you cant determine complexity! +PerceivedComplexity: + Enabled: false + +# Disable requirement of "encoding" headers on files +Encoding: + Enabled: false + +# Increase line length, we're not on VT220s anymore +LineLength: + Max: 255 + # To make it possible to copy or click on URIs in the code, we allow lines + # containing a URI to be longer than Max. + URISchemes: + - http + - https + +# Longer classes aren't _so_ bad +Metrics/ClassLength: + Max: 125 + +# Increase allowed lines in a method. Short methods are good, but 10 lines +# is a bit too low. +MethodLength: + CountComments: false # count full line comments? + Max: 40 + +# Favor explicit over implicit code: don't complain of "redundant returns" +RedundantReturn: + Enabled: false + +# Don't complain about if/unless modifiers. The merit of this is debatable +# and it will likely require building of over-length lines. +IfUnlessModifier: + Enabled: false + +# Raise allowed CyclomaticComplexity & Perceivedto 10. +CyclomaticComplexity: + Max: 15 + +Metrics/PerceivedComplexity: + Max: 15 + +# Don't force a word array unless 5 elements +WordArray: + MinSize: 5 + +# Don't complain about unused block args +UnusedBlockArgument: + Enabled: false + +# allow both hash syntaxes +Style/HashSyntax: + Enabled: false + +# allow final rescue +Style/RescueModifier: + Enabled: false + +Style/AccessorMethodName: + Enabled: false + +# allow larger modules +Metrics/ModuleLength: + Max: 150 + +# disable opinionated doc requirements such as a top-level class comments +Documentation: + Enabled: false + +# allow using parenthases to group an expresion +Lint/ParenthesesAsGroupedExpression: + Enabled: false + +# NumericLiterals: +# Enabled: false + +# allow multiple spaces between methofs and first arg +Style/SpaceBeforeFirstArg: + Enabled: false + +# There are too many non-ruby files that run up against rubocop rules in a cookbook +AllCops: + Include: + - '**/*.rb'