(PDK-1338) Initial import of analytics code from Bolt

.dependency_decisions.yml

@@ -68,3 +68,10 @@
     :why: MIT equivalent
     :versions: []
     :when: 2017-08-11 01:30:32.594531000 Z
+- - :license
+  - facter
+  - Apache 2.0
+  - :who: rodjek
+    :why: https://github.com/puppetlabs/facter/blob/d85f088d74f60859a23e6dbae3f83d918dff504a/LICENSE
+    :versions: []
+    :when: 2019-04-05 00:51:23.372598896 Z

lib/pdk.rb

@@ -1,3 +1,4 @@
+require 'pdk/analytics'
 require 'pdk/answer_file'
 require 'pdk/generate'
 require 'pdk/i18n'
@@ -7,4 +8,10 @@
 require 'pdk/validate'
 require 'pdk/version'
 
-module PDK; end
+module PDK
+  def self.analytics
+    @analytics ||= PDK::Analytics.build_client(
+      logger: PDK.logger,
+    )
+  end
+end

lib/pdk/analytics.rb

@@ -0,0 +1,46 @@
+require 'securerandom'
+require 'pdk/analytics/util'
+require 'pdk/analytics/client/google_analytics'
+require 'pdk/analytics/client/noop'
+
+module PDK
+  module Analytics
+    def self.build_client(logger: ::Logger.new(STDERR))
+      # TODO: PDK-1339
+      config_file = File.expand_path('~/.puppetlabs/bolt/analytics.yaml')
+      config = load_config(config_file)
+
+      if config['disabled'] || ENV['PDK_DISABLE_ANALYTICS']
+        logger.debug 'Analytics opt-out is set, analytics will be disabled'
+        Client::Noop.new(logger)
+      else
+        unless config.key?('user-id')
+          config['user-id'] = SecureRandom.uuid
+          write_config(config_file, config)
+        end
+
+        Client::GoogleAnalytics.new(logger, config['user-id'])
+      end
+    rescue StandardError => e
+      logger.debug "Failed to initialize analytics client, analytics will be disabled: #{e}"
+      Client::Noop.new(logger)
+    end
+
+    # TODO: Extract config handling out of Analytics and pass in the parsed
+    # config instead
+    def self.load_config(filename)
+      # TODO: Catch errors from YAML and File
+      if File.exist?(filename)
+        YAML.safe_load(File.read(filename))
+      else
+        {}
+      end
+    end
+
+    def self.write_config(filename, config)
+      # TODO: Catch errors from FileUtils & File
+      FileUtils.mkdir_p(File.dirname(filename))
+      File.write(filename, config.to_yaml)
+    end
+  end
+end

lib/pdk/analytics/client/google_analytics.rb

@@ -0,0 +1,127 @@
+module PDK
+  module Analytics
+    module Client
+      class GoogleAnalytics
+        PROTOCOL_VERSION  = 1
+        APPLICATION_NAME  = 'pdk'.freeze
+        TRACKING_ID       = 'UA-xxxx-1'.freeze
+        TRACKING_URL      = 'https://google-analytics.com/collect'.freeze
+        CUSTOM_DIMENSIONS = {
+          operating_system: :cd1,
+          output_format:    :cd2,
+        }.freeze
+
+        attr_reader :user_id
+        attr_reader :logger
+
+        def initialize(logger, user_id)
+          # lazy-load expensive gem code
+          require 'concurrent/configuration'
+          require 'concurrent/future'
+          require 'httpclient'
+          require 'locale'
+
+          @http = HTTPClient.new
+          @user_id = user_id
+          @executor = Concurrent.global_io_executor
+          @os = PDK::Analytics::Util.fetch_os_async
+          @logger = logger
+        end
+
+        def screen_view(screen, **kwargs)
+          custom_dimensions = walk_keys(kwargs) do |k|
+            CUSTOM_DIMENSIONS[k] || raise("Unknown analytics key '#{k}'")
+          end
+
+          screen_view_params = {
+            # Type
+            t:  'screenview',
+            # Screen Name
+            cd: screen,
+          }.merge(custom_dimensions)
+
+          submit(base_params.merge(screen_view_params))
+        end
+
+        def event(category, action, label: nil, value: nil, **kwargs)
+          custom_dimensions = walk_keys(kwargs) do |k|
+            CUSTOM_DIMENSIONS[k] || raise("Unknown analytics key '#{k}'")
+          end
+
+          event_params = {
+            # Type
+            t:  'event',
+            # Event Category
+            ec: category,
+            # Event Action
+            ea: action,
+          }.merge(custom_dimensions)
+
+          # Event Label
+          event_params[:el] = label if label
+          # Event Value
+          event_params[:ev] = value if value
+
+          submit(base_params.merge(event_params))
+        end
+
+        def submit(params)
+          # Handle analytics submission in the background to avoid blocking the
+          # app or polluting the log with errors
+          Concurrent::Future.execute(executor: @executor) do
+            logger.debug "Submitting analytics: #{JSON.pretty_generate(params)}"
+            @http.post(TRACKING_URL, params)
+            logger.debug 'Completed analytics submission'
+          end
+        end
+
+        # These parameters have terrible names. See this page for complete documentation:
+        # https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters
+        def base_params
+          {
+            v:   PROTOCOL_VERSION,
+            # Client ID
+            cid: @user_id,
+            # Tracking ID
+            tid: TRACKING_ID,
+            # Application Name
+            an:  APPLICATION_NAME,
+            # Application Version
+            av:  PDK::VERSION,
+            # Anonymize IPs
+            aip: true,
+            # User locale
+            ul:  Locale.current.to_rfc,
+            # Custom Dimension 1 (Operating System)
+            cd1: @os.value,
+          }
+        end
+
+        # If the user is running a very fast command, there may not be time for
+        # analytics submission to complete before the command is finished. In
+        # that case, we give a little buffer for any stragglers to finish up.
+        # 250ms strikes a balance between accomodating slower networks while not
+        # introducing a noticeable "hang".
+        def finish
+          @executor.shutdown
+          @executor.wait_for_termination(0.25)
+        end
+
+        private
+
+        def walk_keys(data, &block)
+          if data.is_a?(Hash)
+            data.each_with_object({}) do |(k, v), acc|
+              v = walk_keys(v, &block)
+              acc[yield(k)] = v
+            end
+          elsif data.is_a?(Array)
+            data.map { |v| walk_keys(v, &block) }
+          else
+            data
+          end
+        end
+      end
+    end
+  end
+end

lib/pdk/analytics/client/noop.rb

@@ -0,0 +1,23 @@
+module PDK
+  module Analytics
+    module Client
+      class Noop
+        attr_reader :logger
+
+        def initialize(logger)
+          @logger = logger
+        end
+
+        def screen_view(screen, **_kwargs)
+          logger.debug "Skipping submission of '#{screen}' screenview because analytics is disabled"
+        end
+
+        def event(category, action, **_kwargs)
+          logger.debug "Skipping submission of '#{category} #{action}' event because analytics is disabled"
+        end
+
+        def finish; end
+      end
+    end
+  end
+end

lib/pdk/analytics/util.rb

@@ -0,0 +1,17 @@
+module PDK
+  module Analytics
+    module Util
+      def self.fetch_os_async
+        require 'concurrent/configuration'
+        require 'concurrent/future'
+
+        Concurrent::Future.execute(executor: :io) do
+          require 'facter'
+          os = Facter.value('os')
+
+          os.nil? ? 'unknown' : "#{os['name']} #{os.fetch('release', {}).fetch('major', '')}".strip
+        end
+      end
+    end
+  end
+end

pdk.gemspec

@@ -37,6 +37,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'hitimes', '1.3.0'
   spec.add_runtime_dependency 'net-ssh', '~> 4.2.0'
 
+  # Analytics dependencies
+  spec.add_runtime_dependency 'httpclient', '~> 2.8.3'
+  spec.add_runtime_dependency 'concurrent-ruby', '~> 1.1.5'
+  spec.add_runtime_dependency 'facter', '~> 2.5.1'
+
   # Used in the pdk-templates
   spec.add_runtime_dependency 'deep_merge', '~> 1.1'
 end

spec/spec_helper.rb

@@ -53,6 +53,7 @@
   # This should catch any tests where we are not mocking out the actual calls to Rubygems.org
   c.before(:each) do
     allow(Gem::SpecFetcher).to receive(:fetcher).and_raise('Unmocked call to Gem::SpecFetcher.fetcher!')
+    ENV['PDK_DISABLE_ANALYTICS'] = 'true'
   end
 
   c.add_setting :root

spec/unit/pdk/analytics/client/google_analytics_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+require 'httpclient'
+require 'concurrent/configuration'
+require 'concurrent/future'
+
+describe PDK::Analytics::Client::GoogleAnalytics do
+  subject(:client) { described_class.new(logger, uuid) }
+
+  let(:uuid) { SecureRandom.uuid }
+  let(:base_params) do
+    {
+      v:   described_class::PROTOCOL_VERSION,
+      an:  'pdk',
+      av:  PDK::VERSION,
+      cid: uuid,
+      tid: 'UA-xxxx-1',
+      ul:  Locale.current.to_rfc,
+      aip: true,
+      cd1: 'CentOS 7',
+    }
+  end
+  let(:mock_httpclient) { instance_double(HTTPClient) }
+  let(:ga_url) { described_class::TRACKING_URL }
+  let(:executor) { Concurrent.new_io_executor }
+  let(:logger) { instance_double(Logger, debug: true) }
+
+  before(:each) do
+    allow(PDK::Analytics::Util).to receive(:fetch_os_async).and_return(instance_double(Concurrent::Future, value: 'CentOS 7'))
+    allow(HTTPClient).to receive(:new).and_return(mock_httpclient)
+    allow(Concurrent).to receive(:global_io_executor).and_return(executor)
+  end
+
+  describe '#screen_view' do
+    after(:each) do
+      client.finish
+    end
+
+    it 'properly formats the screenview' do
+      params = base_params.merge(t: 'screenview', cd: 'job_run')
+
+      expect(mock_httpclient).to receive(:post).with(ga_url, params).and_return(true)
+
+      client.screen_view('job_run')
+    end
+
+    it 'sets custom dimensions correctly' do
+      params = base_params.merge(t: 'screenview', cd: 'job_run', cd1: 'CentOS 7', cd2: 'text')
+
+      expect(mock_httpclient).to receive(:post).with(ga_url, params).and_return(true)
+
+      client.screen_view('job_run', operating_system: 'CentOS 7', output_format: 'text')
+    end
+
+    it 'raises an error if an unknown custom dimension is specified' do
+      expect { client.screen_view('job_run', random_field: 'foo') }.to raise_error(%r{Unknown analytics key})
+    end
+  end
+
+  describe '#event' do
+    after(:each) do
+      client.finish
+    end
+
+    it 'properly formats the event' do
+      params = base_params.merge(t: 'event', ec: 'run', ea: 'task')
+
+      expect(mock_httpclient).to receive(:post).with(ga_url, params).and_return(true)
+
+      client.event('run', 'task')
+    end
+
+    it 'sends the event label if supplied' do
+      params = base_params.merge(t: 'event', ec: 'run', ea: 'task', el: 'happy')
+
+      expect(mock_httpclient).to receive(:post).with(ga_url, params).and_return(true)
+
+      client.event('run', 'task', label: 'happy')
+    end
+
+    it 'sends the event metric if supplied' do
+      params = base_params.merge(t: 'event', ec: 'run', ea: 'task', ev: 12)
+
+      expect(mock_httpclient).to receive(:post).with(ga_url, params).and_return(true)
+
+      client.event('run', 'task', value: 12)
+    end
+  end
+end

spec/unit/pdk/analytics/client/noop_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe PDK::Analytics::Client::Noop do
+  subject(:client) { described_class.new(logger) }
+
+  let(:logger) { instance_double(Logger, debug: true) }
+
+  describe '#screen_view' do
+    it 'does not raise an error' do
+      expect { client.screen_view('job_run') }.not_to raise_error
+    end
+  end
+
+  describe '#event' do
+    it 'does not raise an error' do
+      expect { client.event('run', 'task') }.not_to raise_error
+    end
+
+    context 'with a label' do
+      it 'does not raise an error' do
+        expect { client.event('run', 'task', label: 'happy') }.not_to raise_error
+      end
+    end
+
+    context 'with a value' do
+      it 'does not raise an error' do
+        expect { client.event('run', 'task', value: 12) }.not_to raise_error
+      end
+    end
+  end
+
+  describe '#finish' do
+    it 'does not raise an error' do
+      expect { client.finish }.not_to raise_error
+    end
+  end
+end