Merge pull request #25 from npwalker/use_pg_repack

Add pg_repack as default maintenance strategy

.gitattributes

@@ -0,0 +1,5 @@
+*.rb eol=lf
+*.erb eol=lf
+*.pp eol=lf
+*.sh eol=lf
+*.epp eol=lf

.pdkignore

@@ -22,3 +22,16 @@
 /convert_report.txt
 /update_report.txt
 .DS_Store
+/appveyor.yml
+/.fixtures.yml
+/Gemfile
+/.gitattributes
+/.gitignore
+/.gitlab-ci.yml
+/.pdkignore
+/Rakefile
+/.rspec
+/.rubocop.yml
+/.travis.yml
+/.yardopts
+/spec/

.puppet-lint.rc

@@ -0,0 +1,3 @@
+--relative
+--no-2sp_soft_tabs-check
+--no-arrow_alignment-check

.travis.yml

@@ -1,19 +1,21 @@
 ---
-sudo: false
 dist: trusty
 language: ruby
 cache: bundler
 before_install:
+  - if [ $BUNDLER_VERSION ]; then
+      gem install -v $BUNDLER_VERSION bundler --no-rdoc --no-ri;
+    fi
   - bundle -v
   - rm -f Gemfile.lock
-  - gem update --system
+  - gem update --system $RUBYGEMS_VERSION
   - gem --version
   - bundle -v
 script:
   - 'bundle exec rake $CHECK'
 bundler_args: --without system_tests
 rvm:
-  - 2.5.0
+  - 2.5.1
 env:
   global:
     - BEAKER_PUPPET_COLLECTION=puppet6 PUPPET_GEM_VERSION="~> 6.0"
@@ -28,7 +30,7 @@ matrix:
       env: PUPPET_GEM_VERSION="~> 5.0" CHECK=parallel_spec
       rvm: 2.4.4
     -
-      env: PUPPET_GEM_VERSION="~> 4.0" CHECK=parallel_spec
+      env: PUPPET_GEM_VERSION="~> 4.0" CHECK=parallel_spec RUBYGEMS_VERSION=2.7.8 BUNDLER_VERSION=1.17.3
       rvm: 2.1.9
 branches:
   only:

CHANGELOG.md

@@ -1,3 +1,9 @@
+## Minor Release 0.15.0
+
+- Make pg_repack the default way to perform maintenance [#25](https://github.com/npwalker/pe_databases/pull/25)
+- Start maintaining the reports table which we could not afford to perform a VACUUM FULL on
+- Rename typoed `pe_databases::maintenance::disable_maintenace` parameter to `pe_databases::maintenance::disable_maintenance`
+
 ## Z Release 0.14.2
  - Allow not managing table settings [#21](https://github.com/npwalker/pe_databases/pull/21)
 

Gemfile

@@ -21,7 +21,8 @@ group :development do
   gem "fast_gettext",                                  require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0')
   gem "json_pure", '<= 2.0.1',                         require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0')
   gem "json", '= 1.8.1',                               require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9')
-  gem "json", '<= 2.0.4',                              require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.4.4')
+  gem "json", '= 2.0.4',                               require: false if Gem::Requirement.create('~> 2.4.2').satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "json", '= 2.1.0',                               require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby]
   gem "puppet-module-posix-dev-r#{minor_version}",     require: false, platforms: [:ruby]
   gem "puppet-module-win-default-r#{minor_version}",   require: false, platforms: [:mswin, :mingw, :x64_mingw]

Rakefile

@@ -2,6 +2,7 @@ require 'puppetlabs_spec_helper/rake_tasks'
 require 'puppet-syntax/tasks/puppet-syntax'
 require 'puppet_blacksmith/rake_tasks' if Bundler.rubygems.find_name('puppet-blacksmith').any?
 require 'github_changelog_generator/task' if Bundler.rubygems.find_name('github_changelog_generator').any?
+require 'puppet-strings/tasks' if Bundler.rubygems.find_name('puppet-strings').any?
 
 def changelog_user
   return unless Rake.application.top_level_tasks.include? "changelog"

manifests/maintenance.pp

@@ -1,79 +1,25 @@
 class pe_databases::maintenance (
-  Boolean $disable_maintenace = false,
-  Optional[Integer] $maint_cron_weekday = undef, #DEPRECATED
-  Optional[Integer] $maint_cron_hour    = undef, #DEPRECATED
-  Optional[Integer] $maint_cron_minute  = undef, #DEPRECATED
+  Boolean $disable_maintenance = false,
   String  $logging_directory  = '/var/log/puppetlabs/pe_databases_cron',
   String  $script_directory   = $pe_databases::scripts_dir,
 ){
 
-  $ensure_cron = $disable_maintenace ? {
-    true    => absent,
-    default => present
+  #If the PE Version includes pg_repack (2018.1.7 and 2019.0.2) then use pg_repack and remove the old script and cron jobs
+  if ( versioncmp( '2018.1.7', $facts['pe_server_version']) <= 0 and versioncmp($facts['pe_server_version'], '2019.0.0') < 0 ) {
+    include pe_databases::maintenance::pg_repack
+    class { 'pe_databases::maintenance::vacuum_full':
+      disable_maintenance => true,
+    }
+  } elsif ( versioncmp( '2019.0.2', $facts['pe_server_version']) <= 0 ) {
+    include pe_databases::maintenance::pg_repack
+    class { 'pe_databases::maintenance::vacuum_full':
+      disable_maintenance => true,
+    }
+  } else {
+    include pe_databases::maintenance::vacuum_full
   }
 
   file { $logging_directory :
     ensure => directory,
   }
-
-  $vacuum_script_path = "${script_directory}/vacuum_full_tables.sh"
-
-  file { $vacuum_script_path:
-    ensure => file,
-    source => 'puppet:///modules/pe_databases/vacuum_full_tables.sh',
-    owner  => 'pe-postgres',
-    group  => 'pe-postgres',
-    mode   => '0744',
-  }
-
-  cron { 'VACUUM FULL facts tables' :
-    ensure   => $ensure_cron,
-    user     => 'root',
-    weekday => [2,6],
-    hour     => 4,
-    minute   => 30,
-    command  => "${vacuum_script_path} facts",
-    require  => File[$logging_directory, $script_directory],
-  }
-
-  cron { 'VACUUM FULL catalogs tables' :
-    ensure   => $ensure_cron,
-    user     => 'root',
-    weekday  => [0,4],
-    hour     => 4,
-    minute   => 30,
-    command  => "${vacuum_script_path} catalogs",
-    require  => File[$logging_directory, $script_directory],
-  }
-
-  cron { 'VACUUM FULL other tables' :
-    ensure   => $ensure_cron,
-    user     => 'root',
-    monthday => 20,
-    hour     => 5,
-    minute   => 30,
-    command  => "${vacuum_script_path} other",
-    require  => File[$logging_directory, $script_directory],
-  }
-
-  #Remove old versions of maintenance cron jobs
-  cron { 'Maintain PE databases' :
-    ensure  => absent,
-    user    => 'root',
-    weekday => $maint_cron_weekday,
-    hour    => $maint_cron_hour,
-    minute  => $maint_cron_minute,
-    command => "su - pe-postgres -s /bin/bash -c '/opt/puppetlabs/server/bin/reindexdb --all; /opt/puppetlabs/server/bin/vacuumdb --analyze --verbose --all' > ${logging_directory}/output.log 2> ${logging_directory}/output_error.log",
-    require => File[$logging_directory, $script_directory],
-  }
-
-  if empty($maint_cron_weekday) == false {
-    warning('pe_databases::maintenance::maint_cron_weekday is deprecated and will be removed in a future release')
-  }
-  if empty($maint_cron_hour) == false {
-    warning('pe_databases::maintenance::maint_cron_hour is deprecated and will be removed in a future release')
-  }
-  if empty($maint_cron_minute) == false {
-    warning('pe_databases::maintenance::maint_cron_minute is deprecated and will be removed in a future release')
-  }
 }

manifests/maintenance/pg_repack.pp

@@ -0,0 +1,59 @@
+# A description of what this class does
+#
+# @summary A short summary of the purpose of this class
+#
+# @example
+#   include pe_databases::maintenance::pg_repack
+class pe_databases::maintenance::pg_repack (
+  Boolean $disable_maintenance = $pe_databases::maintenance::disable_maintenance,
+  String  $logging_directory   = $pe_databases::maintenance::logging_directory,
+) {
+
+  $ensure_cron = $disable_maintenance ? {
+    true    => absent,
+    default => present
+  }
+
+  $repack          = 'su - pe-postgres -s /bin/bash -c "/opt/puppetlabs/server/apps/postgresql/bin/pg_repack -d pe-puppetdb'
+  $facts_tables    = '-t factsets -t fact_paths"'
+  $catalogs_tables = '-t catalogs -t catalog_resources -t edges -t certnames"'
+  $other_tables    = '-t producers -t resource_params -t resource_params_cache"'
+  $reports_tables  = '-t reports"'
+  $logging         = "> ${logging_directory}/output.log 2>&1"
+
+  Cron {
+    ensure   => $ensure_cron,
+    user     => 'root',
+    require  => File[$logging_directory],
+  }
+
+  cron { 'pg_repack facts tables' :
+    weekday => [2,6],
+    hour     => 4,
+    minute   => 30,
+    command  => "${repack} ${facts_tables} ${logging}",
+  }
+
+  cron { 'pg_repack catalogs tables' :
+    weekday  => [0,4],
+    hour     => 4,
+    minute   => 30,
+    command  => "${repack} ${catalogs_tables} ${logging}",
+  }
+
+  cron { 'pg_repack other tables' :
+    monthday => 20,
+    hour     => 5,
+    minute   => 30,
+    command  => "${repack} ${other_tables} ${logging}",
+  }
+
+  cron { 'pg_repack reports tables' :
+    ensure   => $ensure_cron,
+    user     => 'root',
+    monthday => 10,
+    hour     => 5,
+    minute   => 30,
+    command  => "${repack} ${reports_tables} ${logging}",
+  }
+}

manifests/maintenance/vacuum_full.pp

@@ -0,0 +1,70 @@
+# A description of what this class does
+#
+# @summary A short summary of the purpose of this class
+#
+# @example
+#   include pe_databases::maintenance::vacuum_full
+class pe_databases::maintenance::vacuum_full (
+  Boolean $disable_maintenance = $pe_databases::maintenance::disable_maintenance,
+  String  $logging_directory   = $pe_databases::maintenance::logging_directory,
+  String  $script_directory    = $pe_databases::maintenance::script_directory,
+){
+
+  $ensure_cron = $disable_maintenance ? {
+    true    => absent,
+    default => present
+  }
+
+  $ensure_vacuum_script = $disable_maintenance ? {
+    true    => absent,
+    default => file
+  }
+
+  $vacuum_script_path = "${script_directory}/vacuum_full_tables.sh"
+
+  file { $vacuum_script_path:
+    ensure => $ensure_vacuum_script,
+    source => 'puppet:///modules/pe_databases/vacuum_full_tables.sh',
+    owner  => 'pe-postgres',
+    group  => 'pe-postgres',
+    mode   => '0744',
+  }
+
+  cron { 'VACUUM FULL facts tables' :
+    ensure   => $ensure_cron,
+    user     => 'root',
+    weekday  => [2,6],
+    hour     => 4,
+    minute   => 30,
+    command  => "${vacuum_script_path} facts",
+    require  => File[$logging_directory, $script_directory],
+  }
+
+  cron { 'VACUUM FULL catalogs tables' :
+    ensure   => $ensure_cron,
+    user     => 'root',
+    weekday  => [0,4],
+    hour     => 4,
+    minute   => 30,
+    command  => "${vacuum_script_path} catalogs",
+    require  => File[$logging_directory, $script_directory],
+  }
+
+  cron { 'VACUUM FULL other tables' :
+    ensure   => $ensure_cron,
+    user     => 'root',
+    monthday => 20,
+    hour     => 5,
+    minute   => 30,
+    command  => "${vacuum_script_path} other",
+    require  => File[$logging_directory, $script_directory],
+  }
+
+  #Remove old versions of maintenance cron jobs
+  cron { 'Maintain PE databases' :
+    ensure  => absent,
+    user    => 'root',
+    command => "su - pe-postgres -s /bin/bash -c '/opt/puppetlabs/server/bin/reindexdb --all; /opt/puppetlabs/server/bin/vacuumdb --analyze --verbose --all' > ${logging_directory}/output.log 2> ${logging_directory}/output_error.log",
+    require => File[$logging_directory, $script_directory],
+  }
+}

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "npwalker-pe_databases",
-  "version": "0.14.2",
+  "version": "0.15.0",
   "author": "npwalker",
   "summary": "A Puppet Module for Backing Up / Maintaining / Tuning Your Puppet Enterprise Databases",
   "license": "Apache-2.0",
@@ -76,7 +76,7 @@
       "version_requirement": ">= 4.0"
     }
   ],
-  "pdk-version": "1.8.0.pre (heads/master-0-g3527c5c)",
+  "pdk-version": "1.9.0.pre (heads/master-0-g24ff96b)",
   "template-url": "https://github.com/puppetlabs/pdk-templates",
-  "template-ref": "heads/master-0-gde71a15"
-}
+  "template-ref": "heads/master-0-ge116657"
+}

spec/classes/maintenance/pg_repack_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe 'pe_databases::maintenance::pg_repack' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:pre_condition) { "class { 'pe_databases': }" }
+      let(:facts) { os_facts }
+
+      it { is_expected.to compile }
+    end
+  end
+end

spec/classes/maintenance/vacuum_full_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe 'pe_databases::maintenance::vacuum_full' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:pre_condition) { "class { 'pe_databases': }" }
+      let(:facts) { os_facts }
+
+      it { is_expected.to compile }
+    end
+  end
+end