diff --git a/.gitignore b/.gitignore index 7181755..f7840e1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ /pkg/ /spec/reports/ /tmp/ -Gemfile.lock .env scaltainer.yml scaltainer.yml.state diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..11aa2f9 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,104 @@ +PATH + remote: . + specs: + scaltainer (0.3.0) + docker-api + dotenv + excon (>= 0.47.0) + kubeclient + prometheus-client + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + coderay (1.1.1) + coveralls (0.8.21) + json (>= 1.8, < 3) + simplecov (~> 0.14.1) + term-ansicolor (~> 1.3) + thor (~> 0.19.4) + tins (~> 1.6) + diff-lcs (1.3) + docile (1.1.5) + docker-api (1.34.2) + excon (>= 0.47.0) + multi_json + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + dotenv (2.7.5) + excon (0.73.0) + ffi (1.12.2) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake + http (4.4.1) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + http-parser (~> 1.2.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.3.0) + http-parser (1.2.1) + ffi-compiler (>= 1.0, < 2.0) + json (2.1.0) + kubeclient (4.6.0) + http (>= 3.0, < 5.0) + recursive-open-struct (~> 1.0, >= 1.0.4) + rest-client (~> 2.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2020.0425) + multi_json (1.14.1) + netrc (0.11.0) + prometheus-client (2.0.0) + public_suffix (4.0.4) + rake (13.0.1) + recursive-open-struct (1.1.1) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rspec (3.6.0) + rspec-core (~> 3.6.0) + rspec-expectations (~> 3.6.0) + rspec-mocks (~> 3.6.0) + rspec-core (3.6.0) + rspec-support (~> 3.6.0) + rspec-expectations (3.6.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.6.0) + rspec-mocks (3.6.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.6.0) + rspec-support (3.6.0) + simplecov (0.14.1) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.0) + term-ansicolor (1.6.0) + tins (~> 1.0) + thor (0.19.4) + tins (1.13.2) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + +PLATFORMS + ruby + +DEPENDENCIES + bundler (~> 1.15) + coderay (~> 1.1) + coveralls (~> 0.8) + rake (>= 12.3.3) + rspec (~> 3.5) + scaltainer! + +BUNDLED WITH + 1.17.3 diff --git a/README.md b/README.md index aaf4dea..4f5da02 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,21 @@ specify the wait time between repetitions using the `-w` parameter in seconds: This will repeatedly call scaltainer every 60 seconds, sleeping in-between. +If you would like to monitor the changes in scaling out and in. You can install +Prometheus and add a configuration parameter pointing to its Push Gateway: + + scaltainer -g prometheus-pushgateway.monitoring.svc.cluster.local:9091 + +Where `prometheus-pushgateway.monitoring.svc.cluster.local:9091` is the address +of the push gateway. For Kubernetes environments the above denotes the gateway service +name (`prometheus-pushgateway`), where it is installed in the namespace called +`monitoring`. Scaltainer will report the following metrics to Prometheus: + +- `rayyan_controller_replicas`: number of replicas scaled (or untouched thereof). +This is labeled by the namespace and controller name, both matching the scaltainer +configuration file. +- `rayyan_scaltainer_ticks`: iterations scaltainer has performed (if `-w` is used) + ## Configuration ### Environment variables @@ -121,7 +136,7 @@ The configuration file (determined by `-f FILE` command line parameter) should b # to get worker metrics endpoint: https://your-app.com/hirefire/$HIREFIRE_TOKEN/info - # optional docker swarm stack name or kubernetes namespace + # optional docker swarm stack name or kubernetes namespace (useful if having push gateway) namespace: mynamespace # list of web services to monitor web_services: diff --git a/exe/scaltainer b/exe/scaltainer index b867c7b..c2ef708 100755 --- a/exe/scaltainer +++ b/exe/scaltainer @@ -3,8 +3,8 @@ require 'scaltainer' begin - configfile, statefile, logger, wait, orchestrator = Scaltainer::Command.parse ARGV - Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator + configfile, statefile, logger, wait, orchestrator, pushgateway = Scaltainer::Command.parse ARGV + Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator, pushgateway rescue => e $stderr.puts e.message $stderr.puts e.backtrace diff --git a/lib/scaltainer/command.rb b/lib/scaltainer/command.rb index 64ce082..c93e255 100644 --- a/lib/scaltainer/command.rb +++ b/lib/scaltainer/command.rb @@ -4,7 +4,7 @@ module Scaltainer class Command def self.parse(args) - configfile, statefile, wait, orchestrator = 'scaltainer.yml', nil, 0, :swarm + configfile, statefile, wait, orchestrator, pushgateway = 'scaltainer.yml', nil, 0, :swarm, nil OptionParser.new do |opts| opts.banner = "Usage: scaltainer [options]" opts.on("-f", "--conf-file FILE", "Specify configuration file (default: scaltainer.yml)") do |file| @@ -19,6 +19,9 @@ def self.parse(args) opts.on("-o", "--orchestrator swarm:kubernetes", [:swarm, :kubernetes], "Specify orchestrator type (default: swarm)") do |o| orchestrator = o end + opts.on("-g", "--prometheus-push-gateway ADDRESS", "Specify prometheus push gateway address in the form of host:port") do |gw| + pushgateway = gw + end opts.on("-v", "--version", "Show version and exit") do puts Scaltainer::VERSION exit 0 @@ -55,7 +58,7 @@ def self.parse(args) logger = Logger.new(STDOUT) logger.level = %w(debug info warn error fatal unknown).find_index((ENV['LOG_LEVEL'] || '').downcase) || 1 - return configfile, statefile, logger, wait, orchestrator + return configfile, statefile, logger, wait, orchestrator, pushgateway end private diff --git a/lib/scaltainer/runner.rb b/lib/scaltainer/runner.rb index 0a889e3..a63134d 100644 --- a/lib/scaltainer/runner.rb +++ b/lib/scaltainer/runner.rb @@ -1,8 +1,10 @@ require "yaml" +require 'prometheus/client' +require 'prometheus/client/push' module Scaltainer class Runner - def initialize(configfile, statefile, logger, wait, orchestrator) + def initialize(configfile, statefile, logger, wait, orchestrator, pushgateway) @orchestrator = orchestrator @logger = logger @default_service_config = { @@ -19,9 +21,12 @@ def initialize(configfile, statefile, logger, wait, orchestrator) endpoint = config["endpoint"] service_type_web = ServiceTypeWeb.new(endpoint) service_type_worker = ServiceTypeWorker.new(endpoint) + register_pushgateway(pushgateway) if pushgateway + namespace = config["namespace"] || config["stack_name"] loop do - run config, state, service_type_web, service_type_worker + run config, state, service_type_web, service_type_worker, namespace save_state statefile, state + sync_pushgateway(namespace, state) if pushgateway sleep wait break if wait == 0 end @@ -29,8 +34,7 @@ def initialize(configfile, statefile, logger, wait, orchestrator) private - def run(config, state, service_type_web, service_type_worker) - namespace = config["namespace"] || config["stack_name"] + def run(config, state, service_type_web, service_type_worker, namespace) iterate_services config["web_services"], namespace, service_type_web, state iterate_services config["worker_services"], namespace, service_type_worker, state end @@ -82,9 +86,11 @@ def process_service(service_name, config, state, namespace, type, metrics) adjusted_replicas = type.adjust_desired_replicas(desired_replicas, config) @logger.debug "Desired number of replicas for #{service.type} #{service.name} is adjusted to #{adjusted_replicas}" replica_diff = adjusted_replicas - current_replicas + state["replicas"] = current_replicas type.yield_to_scale(replica_diff, config, state, metric, service.name, @logger) do scale_out service, current_replicas, adjusted_replicas + state["replicas"] = adjusted_replicas end end @@ -113,5 +119,27 @@ def scale_out(service, current_replicas, desired_replicas) end end + def register_pushgateway(pushgateway) + @registry = Prometheus::Client.registry + @replicas_gauge = @registry.gauge(:rayyan_controller_replicas, docstring: 'Rayyan replicas', labels: [:controller, :namespace]) + @ticks_counter = @registry.counter(:rayyan_scaltainer_ticks, docstring: 'Rayyan Scaltainer ticks', labels: [:namespace]) + + @pushgateway = Prometheus::Client::Push.new("scaltainer", "scaltainer", "http://#{pushgateway}") + end + + def sync_pushgateway(namespace, state) + @logger.debug("Now syncing state #{state} in namespace #{namespace}") + state.each do |service, state| + @replicas_gauge.set(state["replicas"], labels: {namespace: namespace, controller: service}) if state["replicas"] + end + @ticks_counter.increment(labels: {namespace: namespace}) + begin + @pushgateway.add(@registry) + rescue => e + @logger.warn "[#{e.class}] Error pushing metrics to the configured Prometheus Push Gateway: #{e.message}" + end + @logger.info "Pushed metrics successfully to the configured Prometheus Push Gateway" + end + end # class end # module diff --git a/lib/scaltainer/version.rb b/lib/scaltainer/version.rb index 1b1ce47..dffba47 100644 --- a/lib/scaltainer/version.rb +++ b/lib/scaltainer/version.rb @@ -1,3 +1,3 @@ module Scaltainer - VERSION = "0.2.0" + VERSION = "0.3.0" end diff --git a/scaltainer.gemspec b/scaltainer.gemspec index 4a8d18d..f1b7dc4 100644 --- a/scaltainer.gemspec +++ b/scaltainer.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_development_dependency "bundler", "~> 1.15" - spec.add_development_dependency "rake", "~> 10.0" + spec.add_development_dependency "rake", ">= 12.3.3" spec.add_development_dependency 'rspec', '~> 3.5' spec.add_development_dependency 'coderay', '~> 1.1' spec.add_development_dependency 'coveralls', '~> 0.8' @@ -32,4 +32,5 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "docker-api" spec.add_runtime_dependency "kubeclient" spec.add_runtime_dependency "dotenv" + spec.add_runtime_dependency "prometheus-client" end