From 03b980c41db61b5cea952f3b9b87bbb0ddf21fa9 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Tue, 9 Aug 2022 16:44:17 -0500 Subject: [PATCH] Introduce profiles and the promscale profile A profile is an optional "mode" to put the tuner in that tailors the recommendations for a particular workload. If no profile is specified, the tuner is run with a default profile which provides exactly the same behavior as before and corresponds to the typical timescaledb workload. The only other valid profile defined at the moment is "promscale". The profile is specified using either the `--profile` argument or the `TSTUNE_PROFILE` environment variable. The environment variable overrides the command line argument if the command line argument is set to blank. In this way, the environment variable can be passed to `docker run`, and we therefore do not need different docker images per profile. The promscale profile needs to make recommendations for a couple of postgres settings that are measured in units of time. This PR adds functionality to timescaledb-tune to be able to interpret and handle settings measured in time. This was not as straightforward as was hoped because how postgres interprets a setting's value depends upon: 1. whether the setting is a real or integer 2. the default units 3. whether units were specified in the input and whether those units are larger, smaller, or equal to the defaults 4. whether the input was a fractional value With the addition of profiles, it is possible for one profile to provide recommendations for settings which a different profile simply ignores. This PR had to provide a way for a Recommender to signal that it was providing "no recommendation" for a given setting. A SettingsGroup now returns an appropriate Recommender based upon the selected profile. A NullRecommender can be provided in the case that no recommendations are provided for the selected profile and SettingsGroup. A NullRecommender provides no recommendation for all settings passed to it. SettingsGroups can now have a mixture of settings that are measured in time, bytes, or unit-less. The FloatParser implementations and logic had to change to accommodate this. --- README.md | 8 + cmd/timescaledb-tune/main.go | 7 + internal/parse/parse.go | 212 ++++++++++- internal/parse/parse.md | 136 +++++++ internal/parse/parse_test.go | 506 ++++++++++++++++++++++++++- pkg/pgtune/background_writer.go | 83 +++++ pkg/pgtune/background_writer_test.go | 87 +++++ pkg/pgtune/float_parser.go | 42 +++ pkg/pgtune/float_parser_test.go | 77 ++++ pkg/pgtune/memory.go | 5 +- pkg/pgtune/memory_test.go | 17 +- pkg/pgtune/misc.go | 4 +- pkg/pgtune/misc_test.go | 17 +- pkg/pgtune/null_recommender.go | 16 + pkg/pgtune/null_recommender_test.go | 17 + pkg/pgtune/parallel.go | 4 +- pkg/pgtune/parallel_test.go | 24 +- pkg/pgtune/tune.go | 43 ++- pkg/pgtune/tune_test.go | 36 +- pkg/pgtune/wal.go | 109 +++++- pkg/pgtune/wal_test.go | 127 ++++++- pkg/tstune/tune_settings.go | 32 +- pkg/tstune/tune_settings_test.go | 55 --- pkg/tstune/tuner.go | 56 ++- pkg/tstune/tuner_test.go | 87 ++++- 25 files changed, 1607 insertions(+), 200 deletions(-) create mode 100644 internal/parse/parse.md create mode 100644 pkg/pgtune/background_writer.go create mode 100644 pkg/pgtune/background_writer_test.go create mode 100644 pkg/pgtune/float_parser.go create mode 100644 pkg/pgtune/float_parser_test.go create mode 100644 pkg/pgtune/null_recommender.go create mode 100644 pkg/pgtune/null_recommender_test.go diff --git a/README.md b/README.md index 3fc5c50..1b58d4b 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,14 @@ that you accepted from the prompts. #### Other invocations +By default, timescaledb-tune provides recommendations for a typical timescaledb workload. The `--profile` flag can be +used to tailor the recommendations for other workload types. Currently, the only non-default profile is "promscale". +The `TSTUNE_PROFILE` environment variable can also be used to affect this behavior. + +```bash +$ timescaledb-tune --profile promscale +``` + If you want recommendations for a specific amount of memory and/or CPUs: ```bash $ timescaledb-tune --memory="4GB" --cpus=2 diff --git a/cmd/timescaledb-tune/main.go b/cmd/timescaledb-tune/main.go index b5d37a5..cf0a678 100644 --- a/cmd/timescaledb-tune/main.go +++ b/cmd/timescaledb-tune/main.go @@ -45,9 +45,16 @@ func init() { flag.BoolVar(&f.UseColor, "color", true, "Use color in output (works best on dark terminals)") flag.BoolVar(&f.DryRun, "dry-run", false, "Whether to just show the changes without overwriting the configuration file") flag.BoolVar(&f.Restore, "restore", false, "Whether to restore a previously made conf file backup") + flag.StringVar(&f.Profile, "profile", "", "a specific \"mode\" for tailoring recommendations to a special workload type. If blank or unspecified, a default is used unless the TSTUNE_PROFILE environment variable is set. Valid values: \"promscale\"") flag.BoolVar(&showVersion, "version", false, "Show the version of this tool") flag.Parse() + + // the TSTUNE_PROFILE environment variable overrides the --profile flag if the --profile is blank or unset + // this is designed for docker usage + if val := os.Getenv("TSTUNE_PROFILE"); val != "" && f.Profile == "" { + f.Profile = val + } } func main() { diff --git a/internal/parse/parse.go b/internal/parse/parse.go index 318fbdc..302971b 100644 --- a/internal/parse/parse.go +++ b/internal/parse/parse.go @@ -7,6 +7,8 @@ import ( "math" "regexp" "strconv" + "strings" + "time" ) // Byte equivalents (using 1024) of common byte measurements @@ -19,23 +21,93 @@ const ( // Suffixes for byte measurements that are valid to PostgreSQL const ( - TB = "TB" // terabyte - GB = "GB" // gigabyte - MB = "MB" // megabyte - KB = "kB" // kilobyte - B = "" // no unit, therefore: bytes + TB = "TB" // terabyte + GB = "GB" // gigabyte + MB = "MB" // megabyte + KB = "kB" // kilobyte + B = "" // no unit, therefore: bytes ) +// TimeUnit represents valid suffixes for time measurements used by PostgreSQL settings +// https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values +type TimeUnit int64 + +const ( + Microseconds TimeUnit = iota + Milliseconds + Seconds + Minutes + Hours + Days +) + +func (tu TimeUnit) String() string { + switch tu { + case Microseconds: + return "us" + case Milliseconds: + return "ms" + case Seconds: + return "s" + case Minutes: + return "min" + case Hours: + return "h" + case Days: + return "d" + default: + return "unrecognized" + } +} + +func ParseTimeUnit(val string) (TimeUnit, error) { + switch strings.ToLower(val) { + case "us": + return Microseconds, nil + case "ms": + return Milliseconds, nil + case "s": + return Seconds, nil + case "min": + return Minutes, nil + case "h": + return Hours, nil + case "d": + return Days, nil + default: + return TimeUnit(-1), fmt.Errorf(errUnrecognizedTimeUnitsFmt, val) + } +} + +// VarType represents the values from the vartype column of the pg_settings table +type VarType int64 + +const ( + VarTypeReal VarType = iota + VarTypeInteger +) + +func (v VarType) String() string { + switch v { + case VarTypeReal: + return "real" + case VarTypeInteger: + return "integer" + default: + return "unrecognized" + } +} + const ( - errIncorrectFormatFmt = "incorrect PostgreSQL bytes format: '%s'" - errCouldNotParseBytesFmt = "could not parse bytes number: %v" - errCouldNotParseVersionFmt = "unable to parse PG version string: %s" - errUnknownMajorVersionFmt = "unknown major PG version: %s" + errIncorrectBytesFormatFmt = "incorrect PostgreSQL bytes format: '%s'" + errIncorrectTimeFormatFmt = "incorrect PostgreSQL time format: '%s'" + errCouldNotParseBytesFmt = "could not parse bytes number: %v" + errUnrecognizedTimeUnitsFmt = "unrecognized time units: %s" ) var ( - pgBytesRegex = regexp.MustCompile("^([0-9]+)((?:k|M|G|T)B)?$") - pgVersionRegex = regexp.MustCompile("^PostgreSQL ([0-9]+?).([0-9]+?).*") + pgBytesRegex = regexp.MustCompile("^(?:')?([0-9]+)((?:k|M|G|T)B)?(?:')?$") + pgTimeRegex = regexp.MustCompile(`^(?:')?([0-9]+(\.[0-9]+)?)(?:\s*)(us|ms|s|min|h|d)?(?:')?$`) ) func parseIntToFloatUnits(bytes uint64) (float64, string) { @@ -98,7 +170,7 @@ func BytesToPGFormat(bytes uint64) string { func PGFormatToBytes(val string) (uint64, error) { res := pgBytesRegex.FindStringSubmatch(val) if len(res) != 3 { - return 0.0, fmt.Errorf(errIncorrectFormatFmt, val) + return 0.0, fmt.Errorf(errIncorrectBytesFormatFmt, val) } num, err := strconv.ParseInt(res[1], 10, 64) if err != nil { @@ -121,3 +193,119 @@ func PGFormatToBytes(val string) (uint64, error) { } return ret, nil } + +func NextSmallerTimeUnits(units TimeUnit) TimeUnit { + switch units { + case Microseconds: + return Microseconds + case Milliseconds: + return Microseconds + case Seconds: + return Milliseconds + case Minutes: + return Seconds + case Hours: + return Minutes + default: // Days + return Hours + } +} + +func UnitsToDuration(units TimeUnit) time.Duration { + switch units { + case Microseconds: + return time.Microsecond + case Milliseconds: + return time.Millisecond + case Seconds: + return time.Second + case Minutes: + return time.Minute + case Hours: + return time.Hour + case Days: + return 24 * time.Hour + default: + return time.Nanosecond + } +} + +func TimeConversion(fromUnits, toUnits TimeUnit) (float64, error) { + return float64(UnitsToDuration(fromUnits)) / float64(UnitsToDuration(toUnits)), nil +} + +func PGFormatToTime(val string, defaultUnits TimeUnit, vt VarType) (float64, TimeUnit, error) { + // the default units, whether units were specified, and the variable type ALL impact how the value is interpreted + // https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values + + // select unit, vartype, array_agg(name) from pg_settings where unit in ('us', 'ms', 's', 'm', 'h', 'd') group by 1, 2; + + // parse it + res := pgTimeRegex.FindStringSubmatch(val) + if res == nil || len(res) < 2 { + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) + } + + // extract the numeric portion + v, err := strconv.ParseFloat(res[1], 64) + if err != nil { + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) + } + + // extract the units or use the default + unitsWereUnspecified := true + units := defaultUnits + if len(res) >= 4 && res[3] != "" { + unitsWereUnspecified = false + units, err = ParseTimeUnit(res[3]) + if err != nil { + return -1.0, TimeUnit(-1), err + } + } + + convert := func(v float64, units TimeUnit, vt VarType) (float64, TimeUnit, error) { + if _, fract := math.Modf(v); fract < math.Nextafter(0.0, 1.0) { + // not distinguishable as a fractional value + switch vt { + case VarTypeInteger: + return math.Trunc(v), units, nil + case VarTypeReal: + return v, units, nil + } + } else { + // IS a fractional value. it had a decimal component + toUnits := NextSmallerTimeUnits(units) + if err != nil { + return -1.0, TimeUnit(-1), err + } + conv, err := TimeConversion(units, toUnits) + if err != nil { + return -1.0, TimeUnit(-1), err + } + return math.Round(v * conv), toUnits, nil + } + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) + } + + if unitsWereUnspecified { + switch vt { + case VarTypeInteger: + return math.Round(v), units, nil + case VarTypeReal: + return convert(v, units, vt) + } + } else /* units WERE specified */ { + if units == defaultUnits { + switch vt { + case VarTypeInteger: + return math.Round(v), units, nil + case VarTypeReal: + return convert(v, units, vt) + } + } else /* specified units are different from the default units */ { + return convert(v, units, vt) + } + } + // should never get here! + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) +} diff --git a/internal/parse/parse.md b/internal/parse/parse.md new file mode 100644 index 0000000..9586eb4 --- /dev/null +++ b/internal/parse/parse.md @@ -0,0 +1,136 @@ + +https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values + +Postgres settings come in one of several variable types: + +* enum +* string +* bool +* integer +* real + +Some postgres settings are measured in time. They have default units, but postgres can interpret the following units if provided: + +* us - microseconds +* ms - milliseconds +* s - seconds +* min - minutes +* h - hours +* d - days + +In postgres 14, time-based settings come in three flavors: + +* real milliseconds +* integer milliseconds +* integer seconds + +How postgres interprets a setting's value depends upon: + +1. whether the setting is a real or integer +2. the default units +3. whether units were specified in the input and whether those units are larger, smaller, or equal to the defaults +4. whether the input was a fractional value + +statement_timeout is an integer setting with default units of milliseconds. + +``` +-- no units +set statement_timeout to '155'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 155ms │ +└───────────────────┘ + +-- default units +set statement_timeout to '100ms'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 100ms │ +└───────────────────┘ + +-- non-default units +set statement_timeout to '100s'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 100s │ +└───────────────────┘ + +-- non-default units with fraction +set statement_timeout to '100.3s'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 100300ms │ +└───────────────────┘ + +-- default units with fraction +set statement_timeout to '100.7ms'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 101ms │ +└───────────────────┘ + +-- no units with fraction +set statement_timeout to '155.3'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 155ms │ +└───────────────────┘ +``` + +vacuum_cost_delay is a real setting with default units of milliseconds. + +``` +-- no units +set vacuum_cost_delay to '99'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 99ms │ +└───────────────────┘ + +-- default units +set vacuum_cost_delay to '100ms'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 100ms │ +└───────────────────┘ + +-- non-default units +set vacuum_cost_delay to '1000us'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 1ms │ +└───────────────────┘ + +-- non-default units with fraction +set vacuum_cost_delay to '100.3us'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 100.3us │ +└───────────────────┘ + +-- default units with fraction +set vacuum_cost_delay to '50.7ms'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 50700us │ +└───────────────────┘ + +-- no units with fraction +set vacuum_cost_delay to '55.3'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 55300us │ +└───────────────────┘ +``` diff --git a/internal/parse/parse_test.go b/internal/parse/parse_test.go index 3d04953..cc7b4b2 100644 --- a/internal/parse/parse_test.go +++ b/internal/parse/parse_test.go @@ -268,22 +268,22 @@ func TestPGFormatToBytes(t *testing.T) { { desc: "incorrect format #1", input: " 64MB", // no leading spaces - errMsg: fmt.Sprintf(errIncorrectFormatFmt, " 64MB"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, " 64MB"), }, { desc: "incorrect format #2", input: "64b", // bytes not allowed - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "64b"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "64b"), }, { desc: "incorrect format #3", input: "64 GB", // no space between num and units, - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "64 GB"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "64 GB"), }, { desc: "incorrect format #4", input: "-64MB", // negative memory is a no-no - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "-64MB"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "-64MB"), }, { desc: "incorrect format #5", @@ -293,7 +293,7 @@ func TestPGFormatToBytes(t *testing.T) { { desc: "incorrect format #6", input: "5.5" + MB, // decimal memory is a no-no - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "5.5"+MB), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "5.5"+MB), }, { desc: "valid bytes", @@ -340,6 +340,11 @@ func TestPGFormatToBytes(t *testing.T) { input: "2048" + TB, want: 2048 * Terabyte, }, + { + desc: "valid megabytes, wrapped in single-quotes", + input: "'64MB'", + want: 64 * Megabyte, + }, } for _, c := range cases { @@ -361,3 +366,494 @@ func TestPGFormatToBytes(t *testing.T) { } } + +func TestPGFormatToTime(t *testing.T) { + cases := []struct { + desc string + input string + defUnits TimeUnit + varType VarType + wantNum float64 + wantUnits TimeUnit + errMsg string + }{ + { + desc: "set statement_timeout to '13ms';", + input: "13ms", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '13ms'; #2", + input: "'13ms'", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 7;", + input: "7", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 7.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 13;", + input: "13", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 13.5;", + input: "13.5", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 14.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 13.4;", + input: "13.4", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '13.4ms';", + input: "13.4ms", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '13min';", + input: "13min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '13.0min';", + input: "13.0min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '1.5s';", + input: "1.5s", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 1500.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '1.5min';", + input: "1.5min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 90.0, + wantUnits: Seconds, + }, + { + desc: "set statement_timeout to '1.3h';", + input: "1.3h", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 78.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '42.0 min';", + input: "42.0 min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 42.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '42.1 min';", + input: "42.1 min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 2526.0, + wantUnits: Seconds, + }, + { + desc: "set statement_timeout to 'bob';", + input: "bob", + defUnits: Milliseconds, + varType: VarTypeInteger, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "bob"), + }, + { + desc: "set statement_timeout to '42 bob';", + input: "42 bob", + defUnits: Milliseconds, + varType: VarTypeInteger, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "42 bob"), + }, + { + desc: "set vacuum_cost_delay to 250;", + input: "250", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 250.0, + wantUnits: Milliseconds, + }, + { + desc: "set vacuum_cost_delay to 250.0;", + input: "250.0", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 250.0, + wantUnits: Milliseconds, + }, + { + desc: "set vacuum_cost_delay to 1.3;", + input: "1.3", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 1300.0, + wantUnits: Microseconds, + }, + { + desc: "set vacuum_cost_delay to '1.3ms';", + input: "1.3ms", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 1300.0, + wantUnits: Microseconds, + }, + { + desc: "set vacuum_cost_delay to '1300us';", + input: "1300us", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 1300.0, + wantUnits: Microseconds, + }, + { + desc: "37.1 goats", + input: "37.1 goats", + defUnits: Milliseconds, + varType: VarTypeReal, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "37.1 goats"), + }, + { + desc: "37.42.1min", + input: "37.42.1min", + defUnits: Milliseconds, + varType: VarTypeReal, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "37.42.1min"), + }, + } + + for _, c := range cases { + v, u, err := PGFormatToTime(c.input, c.defUnits, c.varType) + if len(c.errMsg) > 0 { // failure cases + if err == nil { + t.Errorf("%s: unexpectedly err is nil: want %s", c.desc, c.errMsg) + } else if got := err.Error(); got != c.errMsg { + t.Errorf("%s: unexpected err msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) + } + } else { + if err != nil { + t.Errorf("%s: unexpected err: got %v", c.desc, err) + } + if got := v; got != c.wantNum { + t.Errorf("%s: incorrect num: got %f want %f", c.desc, got, c.wantNum) + } + if got := u; got != c.wantUnits { + t.Errorf("%s: incorrect units: got %s want %s", c.desc, got, c.wantUnits) + } + } + } +} + +func TestTimeConversion(t *testing.T) { + // test cases generated with the following query + /* + with x(unit, const, val) as + ( + values + ('us', 'Microseconds' , interval '1 microsecond'), + ('ms', 'Milliseconds' , interval '1 millisecond'), + ('s', 'Seconds' , interval '1 second'), + ('min', 'Minutes' , interval '1 minute'), + ('h', 'Hours' , interval '1 hour'), + ('d', 'Days' , interval '24 hours') + ) + select string_agg(format + ( + $${ + desc: "%s -> %s", + from: %s, + to: %s, + want: %s / %s, + }$$, + f.unit, + t.unit, + f.const, + t.const, + extract(epoch from f.val), + extract(epoch from t.val) + ), E',\n' order by f.const, t.const) + from x f + cross join x t + ; + */ + + cases := []struct { + desc string + from TimeUnit + to TimeUnit + want float64 + errMsg string + }{ + { + desc: "d -> d", + from: Days, + to: Days, + want: 86400.000000 / 86400.000000, + }, + { + desc: "d -> h", + from: Days, + to: Hours, + want: 86400.000000 / 3600.000000, + }, + { + desc: "d -> us", + from: Days, + to: Microseconds, + want: 86400.000000 / 0.000001, + }, + { + desc: "d -> ms", + from: Days, + to: Milliseconds, + want: 86400.000000 / 0.001000, + }, + { + desc: "d -> min", + from: Days, + to: Minutes, + want: 86400.000000 / 60.000000, + }, + { + desc: "d -> s", + from: Days, + to: Seconds, + want: 86400.000000 / 1.000000, + }, + { + desc: "h -> d", + from: Hours, + to: Days, + want: 3600.000000 / 86400.000000, + }, + { + desc: "h -> h", + from: Hours, + to: Hours, + want: 3600.000000 / 3600.000000, + }, + { + desc: "h -> us", + from: Hours, + to: Microseconds, + want: 3600.000000 / 0.000001, + }, + { + desc: "h -> ms", + from: Hours, + to: Milliseconds, + want: 3600.000000 / 0.001000, + }, + { + desc: "h -> min", + from: Hours, + to: Minutes, + want: 3600.000000 / 60.000000, + }, + { + desc: "h -> s", + from: Hours, + to: Seconds, + want: 3600.000000 / 1.000000, + }, + { + desc: "us -> d", + from: Microseconds, + to: Days, + want: 0.000001 / 86400.000000, + }, + { + desc: "us -> h", + from: Microseconds, + to: Hours, + want: 0.000001 / 3600.000000, + }, + { + desc: "us -> us", + from: Microseconds, + to: Microseconds, + want: 0.000001 / 0.000001, + }, + { + desc: "us -> ms", + from: Microseconds, + to: Milliseconds, + want: 0.000001 / 0.001000, + }, + { + desc: "us -> min", + from: Microseconds, + to: Minutes, + want: 0.000001 / 60.000000, + }, + { + desc: "us -> s", + from: Microseconds, + to: Seconds, + want: 0.000001 / 1.000000, + }, + { + desc: "ms -> d", + from: Milliseconds, + to: Days, + want: 0.001000 / 86400.000000, + }, + { + desc: "ms -> h", + from: Milliseconds, + to: Hours, + want: 0.001000 / 3600.000000, + }, + { + desc: "ms -> us", + from: Milliseconds, + to: Microseconds, + want: 0.001000 / 0.000001, + }, + { + desc: "ms -> ms", + from: Milliseconds, + to: Milliseconds, + want: 0.001000 / 0.001000, + }, + { + desc: "ms -> min", + from: Milliseconds, + to: Minutes, + want: 0.001000 / 60.000000, + }, + { + desc: "ms -> s", + from: Milliseconds, + to: Seconds, + want: 0.001000 / 1.000000, + }, + { + desc: "min -> d", + from: Minutes, + to: Days, + want: 60.000000 / 86400.000000, + }, + { + desc: "min -> h", + from: Minutes, + to: Hours, + want: 60.000000 / 3600.000000, + }, + { + desc: "min -> us", + from: Minutes, + to: Microseconds, + want: 60.000000 / 0.000001, + }, + { + desc: "min -> ms", + from: Minutes, + to: Milliseconds, + want: 60.000000 / 0.001000, + }, + { + desc: "min -> min", + from: Minutes, + to: Minutes, + want: 60.000000 / 60.000000, + }, + { + desc: "min -> s", + from: Minutes, + to: Seconds, + want: 60.000000 / 1.000000, + }, + { + desc: "s -> d", + from: Seconds, + to: Days, + want: 1.000000 / 86400.000000, + }, + { + desc: "s -> h", + from: Seconds, + to: Hours, + want: 1.000000 / 3600.000000, + }, + { + desc: "s -> us", + from: Seconds, + to: Microseconds, + want: 1.000000 / 0.000001, + }, + { + desc: "s -> ms", + from: Seconds, + to: Milliseconds, + want: 1.000000 / 0.001000, + }, + { + desc: "s -> min", + from: Seconds, + to: Minutes, + want: 1.000000 / 60.000000, + }, + { + desc: "s -> s", + from: Seconds, + to: Seconds, + want: 1.000000 / 1.000000, + }, + } + + for _, c := range cases { + conv, err := TimeConversion(c.from, c.to) + if c.errMsg != "" { + if err != nil { + t.Errorf("%s: unexpectedly err is nil: want %s", c.desc, c.errMsg) + } else if got := err.Error(); got != c.errMsg { + t.Errorf("%s: unexpected err msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) + } + } else { + if err != nil { + t.Errorf("%s: unexpected err: got %v", c.desc, err) + } + if got := conv; got != c.want { + t.Errorf("%s: incorrect conv: got %f want %f", c.desc, got, c.want) + } + } + } +} diff --git a/pkg/pgtune/background_writer.go b/pkg/pgtune/background_writer.go new file mode 100644 index 0000000..b18ede5 --- /dev/null +++ b/pkg/pgtune/background_writer.go @@ -0,0 +1,83 @@ +package pgtune + +import "github.com/timescale/timescaledb-tune/internal/parse" + +const ( + BgwriterDelayKey = "bgwriter_delay" + BgwriterLRUMaxPagesKey = "bgwriter_lru_maxpages" + + promscaleDefaultBgwriterDelay = "10ms" + promscaleDefaultBgwriterLRUMaxPages = "100000" +) + +// BgwriterLabel is the label used to refer to the background writer settings group +const BgwriterLabel = "background writer" + +var BgwriterKeys = []string{ + BgwriterDelayKey, + BgwriterLRUMaxPagesKey, +} + +// PromscaleBgwriterRecommender gives recommendations for the background writer for the promscale profile +type PromscaleBgwriterRecommender struct{} + +// IsAvailable returns whether this Recommender is usable given the system resources. Always true. +func (r *PromscaleBgwriterRecommender) IsAvailable() bool { + return true +} + +// Recommend returns the recommended PostgreSQL formatted value for the conf +// file for a given key. +func (r *PromscaleBgwriterRecommender) Recommend(key string) string { + switch key { + case BgwriterDelayKey: + return promscaleDefaultBgwriterDelay + case BgwriterLRUMaxPagesKey: + return promscaleDefaultBgwriterLRUMaxPages + default: + return NoRecommendation + } +} + +// BgwriterSettingsGroup is the SettingsGroup to represent settings that affect the background writer. +type BgwriterSettingsGroup struct { + totalMemory uint64 + cpus int + maxConns uint64 +} + +// Label should always return the value BgwriterLabel. +func (sg *BgwriterSettingsGroup) Label() string { return BgwriterLabel } + +// Keys should always return the BgwriterKeys slice. +func (sg *BgwriterSettingsGroup) Keys() []string { return BgwriterKeys } + +// GetRecommender should return a new Recommender. +func (sg *BgwriterSettingsGroup) GetRecommender(profile Profile) Recommender { + switch profile { + case PromscaleProfile: + return &PromscaleBgwriterRecommender{} + default: + return &NullRecommender{} + } +} + +type BgwriterFloatParser struct{} + +func (v *BgwriterFloatParser) ParseFloat(key string, s string) (float64, error) { + switch key { + case BgwriterDelayKey: + val, units, err := parse.PGFormatToTime(s, parse.Milliseconds, parse.VarTypeInteger) + if err != nil { + return val, err + } + conv, err := parse.TimeConversion(units, parse.Milliseconds) + if err != nil { + return val, err + } + return val * conv, nil + default: + bfp := &numericFloatParser{} + return bfp.ParseFloat(key, s) + } +} diff --git a/pkg/pgtune/background_writer_test.go b/pkg/pgtune/background_writer_test.go new file mode 100644 index 0000000..f5cc0b1 --- /dev/null +++ b/pkg/pgtune/background_writer_test.go @@ -0,0 +1,87 @@ +package pgtune + +import ( + "fmt" + "testing" + + "github.com/timescale/timescaledb-tune/internal/parse" +) + +func TestBgwriterSettingsGroup_GetRecommender(t *testing.T) { + cases := []struct { + profile Profile + recommender string + }{ + {DefaultProfile, "*pgtune.NullRecommender"}, + {PromscaleProfile, "*pgtune.PromscaleBgwriterRecommender"}, + } + + sg := BgwriterSettingsGroup{} + for _, k := range cases { + r := sg.GetRecommender(k.profile) + y := fmt.Sprintf("%T", r) + if y != k.recommender { + t.Errorf("Expected to get a %s using the %s profile but got %s", k.recommender, k.profile, y) + } + } +} + +func TestBgwriterSettingsGroupRecommend(t *testing.T) { + sg := BgwriterSettingsGroup{} + + // the default profile should provide no recommendations + r := sg.GetRecommender(DefaultProfile) + if val := r.Recommend(BgwriterDelayKey); val != NoRecommendation { + t.Errorf("Expected no recommendation for key %s but got %s", BgwriterDelayKey, val) + } + if val := r.Recommend(BgwriterLRUMaxPagesKey); val != NoRecommendation { + t.Errorf("Expected no recommendation for key %s but got %s", BgwriterLRUMaxPagesKey, val) + } + + // the promscale profile should have recommendations + r = sg.GetRecommender(PromscaleProfile) + if val := r.Recommend(BgwriterDelayKey); val != promscaleDefaultBgwriterDelay { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterDelay, BgwriterDelayKey, val) + } + if val := r.Recommend(BgwriterLRUMaxPagesKey); val != promscaleDefaultBgwriterLRUMaxPages { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterLRUMaxPages, BgwriterLRUMaxPagesKey, val) + } +} + +func TestPromscaleBgwriterRecommender(t *testing.T) { + r := PromscaleBgwriterRecommender{} + if !r.IsAvailable() { + t.Error("PromscaleBgwriterRecommender should always be available") + } + if val := r.Recommend(BgwriterDelayKey); val != promscaleDefaultBgwriterDelay { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterDelay, BgwriterDelayKey, val) + } + if val := r.Recommend(BgwriterLRUMaxPagesKey); val != promscaleDefaultBgwriterLRUMaxPages { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterLRUMaxPages, BgwriterLRUMaxPagesKey, val) + } +} + +func TestBgwriterFloatParserParseFloat(t *testing.T) { + v := &BgwriterFloatParser{} + + s := "100" + want := 100.0 + got, err := v.ParseFloat(BgwriterLRUMaxPagesKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } + + s = "33" + parse.Minutes.String() + conversion, _ := parse.TimeConversion(parse.Minutes, parse.Milliseconds) + want = 33.0 * conversion + got, err = v.ParseFloat(BgwriterDelayKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } +} diff --git a/pkg/pgtune/float_parser.go b/pkg/pgtune/float_parser.go new file mode 100644 index 0000000..95a70aa --- /dev/null +++ b/pkg/pgtune/float_parser.go @@ -0,0 +1,42 @@ +package pgtune + +import ( + "strconv" + + "github.com/timescale/timescaledb-tune/internal/parse" +) + +type FloatParser interface { + ParseFloat(string, string) (float64, error) +} + +type bytesFloatParser struct{} + +func (v *bytesFloatParser) ParseFloat(key string, s string) (float64, error) { + temp, err := parse.PGFormatToBytes(s) + return float64(temp), err +} + +type numericFloatParser struct{} + +func (v *numericFloatParser) ParseFloat(key string, s string) (float64, error) { + return strconv.ParseFloat(s, 64) +} + +// GetFloatParser returns the correct FloatParser for a given Recommender. +func GetFloatParser(r Recommender) FloatParser { + switch r.(type) { + case *MemoryRecommender: + return &bytesFloatParser{} + case *WALRecommender: + return &WALFloatParser{} + case *PromscaleWALRecommender: + return &WALFloatParser{} + case *PromscaleBgwriterRecommender: + return &BgwriterFloatParser{} + case *ParallelRecommender: + return &numericFloatParser{} + default: + return &numericFloatParser{} + } +} diff --git a/pkg/pgtune/float_parser_test.go b/pkg/pgtune/float_parser_test.go new file mode 100644 index 0000000..390e9cf --- /dev/null +++ b/pkg/pgtune/float_parser_test.go @@ -0,0 +1,77 @@ +package pgtune + +import ( + "testing" + + "github.com/timescale/timescaledb-tune/internal/parse" +) + +func TestBytesFloatParserParseFloat(t *testing.T) { + s := "8" + parse.GB + want := float64(8 * parse.Gigabyte) + v := &bytesFloatParser{} + got, err := v.ParseFloat("foo", s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } +} + +func TestNumericFloatParserParseFloat(t *testing.T) { + s := "8.245" + want := 8.245 + v := &numericFloatParser{} + got, err := v.ParseFloat("foo", s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } +} + +func TestGetFloatParser(t *testing.T) { + switch x := (GetFloatParser(&MemoryRecommender{})).(type) { + case *bytesFloatParser: + default: + t.Errorf("wrong validator type for MemoryRecommender: got %T", x) + } + + switch x := (GetFloatParser(&WALRecommender{})).(type) { + case *WALFloatParser: + default: + t.Errorf("wrong validator type for WALRecommender: got %T", x) + } + + switch x := (GetFloatParser(&PromscaleWALRecommender{})).(type) { + case *WALFloatParser: + default: + t.Errorf("wrong validator type for PromscaleWALRecommender: got %T", x) + } + + switch x := (GetFloatParser(&ParallelRecommender{})).(type) { + case *numericFloatParser: + default: + t.Errorf("wrong validator type for ParallelRecommender: got %T", x) + } + + switch x := (GetFloatParser(&PromscaleBgwriterRecommender{})).(type) { + case *BgwriterFloatParser: + default: + t.Errorf("wrong validator type for PromscaleBgwriterRecommender: got %T", x) + } + + switch x := (GetFloatParser(&MiscRecommender{})).(type) { + case *numericFloatParser: + default: + t.Errorf("wrong validator type for MiscRecommender: got %T", x) + } + + switch x := (GetFloatParser(&NullRecommender{})).(type) { + case *numericFloatParser: + default: + t.Errorf("wrong validator type for NullRecommender: got %T", x) + } +} diff --git a/pkg/pgtune/memory.go b/pkg/pgtune/memory.go index e4125d5..5468963 100644 --- a/pkg/pgtune/memory.go +++ b/pkg/pgtune/memory.go @@ -1,7 +1,6 @@ package pgtune import ( - "fmt" "math" "runtime" @@ -89,7 +88,7 @@ func (r *MemoryRecommender) Recommend(key string) string { } } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -126,6 +125,6 @@ func (sg *MemorySettingsGroup) Label() string { return MemoryLabel } func (sg *MemorySettingsGroup) Keys() []string { return MemoryKeys } // GetRecommender should return a new MemoryRecommender. -func (sg *MemorySettingsGroup) GetRecommender() Recommender { +func (sg *MemorySettingsGroup) GetRecommender(profile Profile) Recommender { return NewMemoryRecommender(sg.totalMemory, sg.cpus, sg.maxConns) } diff --git a/pkg/pgtune/memory_test.go b/pkg/pgtune/memory_test.go index f0e48bd..fa38a7c 100644 --- a/pkg/pgtune/memory_test.go +++ b/pkg/pgtune/memory_test.go @@ -214,16 +214,11 @@ func TestMemoryRecommenderRecommend(t *testing.T) { } } -func TestMemoryRecommenderRecommendPanic(t *testing.T) { - func() { - r := NewMemoryRecommender(1, 1, 1) - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestMemoryRecommenderNoRecommendation(t *testing.T) { + r := NewMemoryRecommender(1, 1, 1) + if r.Recommend("foo") != NoRecommendation { + t.Error("Recommendation was provided when there should have been none") + } } func TestMemorySettingsGroup(t *testing.T) { @@ -236,7 +231,7 @@ func TestMemorySettingsGroup(t *testing.T) { config.maxConns = conns sg := GetSettingsGroup(MemoryLabel, config) - testSettingGroup(t, sg, matrix, MemoryLabel, MemoryKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, MemoryLabel, MemoryKeys) } } } diff --git a/pkg/pgtune/misc.go b/pkg/pgtune/misc.go index 1fceb6e..92fc8b8 100644 --- a/pkg/pgtune/misc.go +++ b/pkg/pgtune/misc.go @@ -141,7 +141,7 @@ func (r *MiscRecommender) Recommend(key string) string { } else if key == EffectiveIOKey { val = getEffectiveIOConcurrency(r.pgMajorVersion) } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -165,6 +165,6 @@ func (sg *MiscSettingsGroup) Keys() []string { } // GetRecommender should return a new MiscRecommender. -func (sg *MiscSettingsGroup) GetRecommender() Recommender { +func (sg *MiscSettingsGroup) GetRecommender(profile Profile) Recommender { return NewMiscRecommender(sg.totalMemory, sg.maxConns, sg.pgMajorVersion) } diff --git a/pkg/pgtune/misc_test.go b/pkg/pgtune/misc_test.go index 1369261..65a9929 100644 --- a/pkg/pgtune/misc_test.go +++ b/pkg/pgtune/misc_test.go @@ -179,16 +179,11 @@ func TestMiscRecommenderRecommend(t *testing.T) { } } -func TestMiscRecommenderRecommendPanic(t *testing.T) { - func() { - r := &MiscRecommender{} - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestMiscRecommenderNoRecommendation(t *testing.T) { + r := &MiscRecommender{} + if r.Recommend("foo") != NoRecommendation { + t.Error("Recommendation was provided when there should have been none") + } } func TestMiscSettingsGroup(t *testing.T) { @@ -200,7 +195,7 @@ func TestMiscSettingsGroup(t *testing.T) { } sg := GetSettingsGroup(MiscLabel, config) - testSettingGroup(t, sg, matrix, MiscLabel, MiscKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, MiscLabel, MiscKeys) } } } diff --git a/pkg/pgtune/null_recommender.go b/pkg/pgtune/null_recommender.go new file mode 100644 index 0000000..cdbad70 --- /dev/null +++ b/pkg/pgtune/null_recommender.go @@ -0,0 +1,16 @@ +package pgtune + +// NullRecommender is a Recommender that returns NoRecommendation for all keys +type NullRecommender struct { +} + +// IsAvailable returns whether this Recommender is usable given the system resources. Always true. +func (r *NullRecommender) IsAvailable() bool { + return true +} + +// Recommend returns the recommended PostgreSQL formatted value for the conf +// file for a given key. +func (r *NullRecommender) Recommend(key string) string { + return NoRecommendation +} diff --git a/pkg/pgtune/null_recommender_test.go b/pkg/pgtune/null_recommender_test.go new file mode 100644 index 0000000..a7764b8 --- /dev/null +++ b/pkg/pgtune/null_recommender_test.go @@ -0,0 +1,17 @@ +package pgtune + +import ( + "fmt" + "testing" +) + +func TestNullRecommender_Recommend(t *testing.T) { + r := &NullRecommender{} + // NullRecommender should ALWAYS return NoRecommendation + for i := 0; i < 1000; i++ { + key := fmt.Sprintf("key%d", i) + if val := r.Recommend(key); val != NoRecommendation { + t.Errorf("Expected no recommendation for key %s but got %s", key, val) + } + } +} diff --git a/pkg/pgtune/parallel.go b/pkg/pgtune/parallel.go index 40f6066..c8057a3 100644 --- a/pkg/pgtune/parallel.go +++ b/pkg/pgtune/parallel.go @@ -69,7 +69,7 @@ func (r *ParallelRecommender) Recommend(key string) string { } else if key == MaxBackgroundWorkers { val = fmt.Sprintf("%d", r.maxBGWorkers) } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -93,6 +93,6 @@ func (sg *ParallelSettingsGroup) Keys() []string { } // GetRecommender should return a new ParallelRecommender. -func (sg *ParallelSettingsGroup) GetRecommender() Recommender { +func (sg *ParallelSettingsGroup) GetRecommender(profile Profile) Recommender { return NewParallelRecommender(sg.cpus, sg.maxBGWorkers) } diff --git a/pkg/pgtune/parallel_test.go b/pkg/pgtune/parallel_test.go index f400572..e14bc2c 100644 --- a/pkg/pgtune/parallel_test.go +++ b/pkg/pgtune/parallel_test.go @@ -99,18 +99,14 @@ func TestParallelRecommenderRecommend(t *testing.T) { } } -func TestParallelRecommenderRecommendPanics(t *testing.T) { - // test invalid key panic - func() { - r := &ParallelRecommender{5, MaxBackgroundWorkersDefault} - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestParallelRecommenderNoRecommendation(t *testing.T) { + r := &ParallelRecommender{5, MaxBackgroundWorkersDefault} + if r.Recommend("foo") != NoRecommendation { + t.Error("Recommendation was provided when there should have been none") + } +} +func TestParallelRecommenderRecommendPanics(t *testing.T) { // test invalid CPU panic func() { defer func() { @@ -146,7 +142,7 @@ func TestParallelSettingsGroup(t *testing.T) { if got := len(sg.Keys()); got != keyCount-1 { t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion96, got, keyCount-1) } - testSettingGroup(t, sg, matrix, ParallelLabel, ParallelKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) // PG10 adds a key config.PGMajorVersion = pgutils.MajorVersion10 @@ -154,14 +150,14 @@ func TestParallelSettingsGroup(t *testing.T) { if got := len(sg.Keys()); got != keyCount { t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion10, got, keyCount) } - testSettingGroup(t, sg, matrix, ParallelLabel, ParallelKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) config.PGMajorVersion = pgutils.MajorVersion11 sg = GetSettingsGroup(ParallelLabel, config) if got := len(sg.Keys()); got != keyCount { t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion11, got, keyCount) } - testSettingGroup(t, sg, matrix, ParallelLabel, ParallelKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) } } diff --git a/pkg/pgtune/tune.go b/pkg/pgtune/tune.go index 0ab1eb2..2e6a6de 100644 --- a/pkg/pgtune/tune.go +++ b/pkg/pgtune/tune.go @@ -3,14 +3,51 @@ // for groups of settings in a PostgreSQL conf file. package pgtune -import "fmt" +import ( + "fmt" + "strings" +) const ( osWindows = "windows" errMaxConnsTooLowFmt = "maxConns must be 0 OR >= %d: got %d" errMaxBGWorkersTooLowFmt = "maxBGWorkers must be >= %d: got %d" + errUnrecognizedProfile = "unrecognized profile: %s" +) + +// Profile is a specific "mode" in which timescaledb-tune can be run to provide recommendations tailored to a +// special workload type, e.g. "promscale" +type Profile int64 + +const ( + DefaultProfile Profile = iota + PromscaleProfile ) +func ParseProfile(s string) (Profile, error) { + switch strings.ToLower(s) { + case "": + return DefaultProfile, nil + case "promscale": + return PromscaleProfile, nil + default: + return DefaultProfile, fmt.Errorf(errUnrecognizedProfile, s) + } +} + +func (p Profile) String() string { + switch p { + case DefaultProfile: + return "" + case PromscaleProfile: + return "promscale" + default: + return "unrecognized" + } +} + +const NoRecommendation = "" + // Recommender is an interface that gives setting recommendations for a given // key, usually grouped by logical settings groups (e.g. MemoryRecommender for memory settings). type Recommender interface { @@ -29,7 +66,7 @@ type SettingsGroup interface { // Keys are the parameter names/keys as they appear in the PostgreSQL conf file, e.g. "shared_buffers". Keys() []string // GetRecommender returns the Recommender that should be used for this group of settings. - GetRecommender() Recommender + GetRecommender(Profile) Recommender } // SystemConfig represents a system's resource configuration, to be used when generating @@ -71,6 +108,8 @@ func GetSettingsGroup(label string, config *SystemConfig) SettingsGroup { return &ParallelSettingsGroup{config.PGMajorVersion, config.CPUs, config.MaxBGWorkers} case label == WALLabel: return &WALSettingsGroup{config.Memory, config.WALDiskSize} + case label == BgwriterLabel: + return &BgwriterSettingsGroup{} case label == MiscLabel: return &MiscSettingsGroup{config.Memory, config.maxConns, config.PGMajorVersion} } diff --git a/pkg/pgtune/tune_test.go b/pkg/pgtune/tune_test.go index 1b649e3..be70d7b 100644 --- a/pkg/pgtune/tune_test.go +++ b/pkg/pgtune/tune_test.go @@ -3,6 +3,7 @@ package pgtune import ( "fmt" "math/rand" + "strings" "testing" ) @@ -80,7 +81,7 @@ func TestNewSystemConfig(t *testing.T) { } func TestGetSettingsGroup(t *testing.T) { - okLabels := []string{MemoryLabel, ParallelLabel, WALLabel, MiscLabel} + okLabels := []string{MemoryLabel, ParallelLabel, WALLabel, BgwriterLabel, MiscLabel} config := getDefaultTestSystemConfig(t) for _, label := range okLabels { sg := GetSettingsGroup(label, config) @@ -109,6 +110,9 @@ func TestGetSettingsGroup(t *testing.T) { if x.walDiskSize != config.WALDiskSize { t.Errorf("WAL group incorrect (wal disk): got %d want %d", x.walDiskSize, config.WALDiskSize) } + case *BgwriterSettingsGroup: + // nothing to check here + continue case *MiscSettingsGroup: if x.totalMemory != config.Memory { t.Errorf("Misc group incorrect (memory): got %d want %d", x.totalMemory, config.Memory) @@ -132,7 +136,7 @@ func TestGetSettingsGroup(t *testing.T) { }() } -func testSettingGroup(t *testing.T, sg SettingsGroup, cases map[string]string, wantLabel string, wantKeys []string) { +func testSettingGroup(t *testing.T, sg SettingsGroup, profile Profile, cases map[string]string, wantLabel string, wantKeys []string) { t.Helper() // No matter how many calls, all calls should return the same @@ -149,7 +153,7 @@ func testSettingGroup(t *testing.T, sg SettingsGroup, cases map[string]string, w } } - r := sg.GetRecommender() + r := sg.GetRecommender(profile) testRecommender(t, r, sg.Keys(), cases) } @@ -168,7 +172,31 @@ func testRecommender(t *testing.T, r Recommender, keys []string, wants map[strin for _, key := range keys { want := wants[key] if got := r.Recommend(key); got != want { - t.Errorf("%v: incorrect result for key %s: got\n%s\nwant\n%s", r, key, got, want) + t.Errorf("%T: incorrect result for key %s: got\n%s\nwant\n%s", r, key, got, want) } } } + +func TestParseProfile(t *testing.T) { + cases := []struct { + input string + expected Profile + }{ + {input: DefaultProfile.String(), expected: DefaultProfile}, + {input: PromscaleProfile.String(), expected: PromscaleProfile}, + {input: strings.ToUpper(PromscaleProfile.String()), expected: PromscaleProfile}, + } + for _, kase := range cases { + actual, err := ParseProfile(kase.input) + if err != nil { + t.Errorf("expected %v for input %s but got an error: %v", kase.expected, kase.input, err) + } + if actual != kase.expected { + t.Errorf("expected %v for input %s but got %v", kase.expected, kase.input, actual) + } + } + + if actual, err := ParseProfile("garbage"); err == nil { + t.Errorf("expected to get an error for unrecognized input, but did not. got %v", actual) + } +} diff --git a/pkg/pgtune/wal.go b/pkg/pgtune/wal.go index 450e66e..d9ea7e4 100644 --- a/pkg/pgtune/wal.go +++ b/pkg/pgtune/wal.go @@ -1,21 +1,22 @@ package pgtune import ( - "fmt" - "github.com/timescale/timescaledb-tune/internal/parse" ) // Keys in the conf file that are tuned related to the WAL const ( - WALBuffersKey = "wal_buffers" - MinWALKey = "min_wal_size" - MaxWALKey = "max_wal_size" - - walMaxDiskPct = 60 // max_wal_size should be 60% of the WAL disk - walBuffersThreshold = 2 * parse.Gigabyte - walBuffersDefault = 16 * parse.Megabyte - defaultMaxWALBytes = 1 * parse.Gigabyte + WALBuffersKey = "wal_buffers" + MinWALKey = "min_wal_size" + MaxWALKey = "max_wal_size" + CheckpointTimeoutKey = "checkpoint_timeout" + + walMaxDiskPct = 60 // max_wal_size should be 60% of the WAL disk + walBuffersThreshold = 2 * parse.Gigabyte + walBuffersDefault = 16 * parse.Megabyte + defaultMaxWALBytes = 1 * parse.Gigabyte + promscaleDefaultMaxWALBytes = 4 * parse.Gigabyte + promscaleDefaultCheckpointTimeout = "900" // 15 minutes expressed in seconds ) // WALLabel is the label used to refer to the WAL settings group @@ -26,6 +27,7 @@ var WALKeys = []string{ WALBuffersKey, MinWALKey, MaxWALKey, + CheckpointTimeoutKey, } // WALRecommender gives recommendations for WALKeys based on system resources @@ -66,7 +68,7 @@ func (r *WALRecommender) Recommend(key string) string { temp := r.calcMaxWALBytes() val = parse.BytesToPGFormat(temp) } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -102,6 +104,87 @@ func (sg *WALSettingsGroup) Label() string { return WALLabel } func (sg *WALSettingsGroup) Keys() []string { return WALKeys } // GetRecommender should return a new WALRecommender. -func (sg *WALSettingsGroup) GetRecommender() Recommender { - return NewWALRecommender(sg.totalMemory, sg.walDiskSize) +func (sg *WALSettingsGroup) GetRecommender(profile Profile) Recommender { + switch profile { + case PromscaleProfile: + return NewPromscaleWALRecommender(sg.totalMemory, sg.walDiskSize) + default: + return NewWALRecommender(sg.totalMemory, sg.walDiskSize) + } +} + +// PromscaleWALRecommender gives recommendations for WALKeys based on system resources +type PromscaleWALRecommender struct { + WALRecommender +} + +// NewPromscaleWALRecommender returns a PromscaleWALRecommender that recommends based on the given +// totalMemory bytes. +func NewPromscaleWALRecommender(totalMemory, walDiskSize uint64) *PromscaleWALRecommender { + return &PromscaleWALRecommender{ + WALRecommender: WALRecommender{ + totalMemory: totalMemory, + walDiskSize: walDiskSize, + }, + } +} + +// IsAvailable returns whether this Recommender is usable given the system resources. Always true. +func (r *PromscaleWALRecommender) IsAvailable() bool { + return true +} + +// Recommend returns the recommended PostgreSQL formatted value for the conf +// file for a given key. +func (r *PromscaleWALRecommender) Recommend(key string) string { + switch key { + case MinWALKey: + temp := r.promscaleCalcMaxWALBytes() / 2 + return parse.BytesToPGFormat(temp) + case MaxWALKey: + temp := r.promscaleCalcMaxWALBytes() + return parse.BytesToPGFormat(temp) + case CheckpointTimeoutKey: + return promscaleDefaultCheckpointTimeout + default: + return r.WALRecommender.Recommend(key) + } +} + +func (r *WALRecommender) promscaleCalcMaxWALBytes() uint64 { + // If disk size is not given, just use default + if r.walDiskSize == 0 { + return promscaleDefaultMaxWALBytes + } + + // With size given, we want to take up at most walMaxDiskPct, to give + // additional room for safety. + max := uint64(r.walDiskSize*walMaxDiskPct) / 100 + + // WAL segments are 16MB, so it doesn't make sense not to round + // up to the nearest 16MB boundary. + if max%(16*parse.Megabyte) != 0 { + max = (max/(16*parse.Megabyte) + 1) * 16 * parse.Megabyte + } + return max +} + +type WALFloatParser struct{} + +func (v *WALFloatParser) ParseFloat(key string, s string) (float64, error) { + switch key { + case CheckpointTimeoutKey: + val, units, err := parse.PGFormatToTime(s, parse.Milliseconds, parse.VarTypeInteger) + if err != nil { + return val, err + } + conv, err := parse.TimeConversion(units, parse.Milliseconds) + if err != nil { + return val, err + } + return val * conv, nil + default: + bfp := &bytesFloatParser{} + return bfp.ParseFloat(key, s) + } } diff --git a/pkg/pgtune/wal_test.go b/pkg/pgtune/wal_test.go index 139d5cb..ee98aec 100644 --- a/pkg/pgtune/wal_test.go +++ b/pkg/pgtune/wal_test.go @@ -1,6 +1,7 @@ package pgtune import ( + "fmt" "math/rand" "testing" @@ -29,10 +30,20 @@ var walDiskToMaxBytes = map[uint64]uint64{ walDiskDivideEvenly: 5280 * parse.Megabyte, } +var promscaleWALDiskToMaxBytes = map[uint64]uint64{ + walDiskUnset: promscaleDefaultMaxWALBytes, + walDiskDivideUnevenly: 4928 * parse.Megabyte, // nearest 16MB segment + walDiskDivideEvenly: 5280 * parse.Megabyte, +} + // walSettingsMatrix stores the test cases for WALRecommender along with the // expected values for WAL keys. var walSettingsMatrix = map[uint64]map[uint64]map[string]string{} +// walSettingsMatrix stores the test cases for WALRecommender along with the +// expected values for WAL keys. +var promscaleWalSettingsMatrix = map[uint64]map[uint64]map[string]string{} + func init() { for memory, walBuffers := range memoryToWALBuffers { walSettingsMatrix[memory] = make(map[uint64]map[string]string) @@ -41,6 +52,37 @@ func init() { walSettingsMatrix[memory][walSize][MinWALKey] = parse.BytesToPGFormat(walDiskToMaxBytes[walSize] / 2) walSettingsMatrix[memory][walSize][MaxWALKey] = parse.BytesToPGFormat(walDiskToMaxBytes[walSize]) walSettingsMatrix[memory][walSize][WALBuffersKey] = parse.BytesToPGFormat(walBuffers) + walSettingsMatrix[memory][walSize][CheckpointTimeoutKey] = NoRecommendation + } + } + + for memory, walBuffers := range memoryToWALBuffers { + promscaleWalSettingsMatrix[memory] = make(map[uint64]map[string]string) + for walSize := range walDiskToMaxBytes { + promscaleWalSettingsMatrix[memory][walSize] = make(map[string]string) + promscaleWalSettingsMatrix[memory][walSize][MinWALKey] = parse.BytesToPGFormat(promscaleWALDiskToMaxBytes[walSize] / 2) + promscaleWalSettingsMatrix[memory][walSize][MaxWALKey] = parse.BytesToPGFormat(promscaleWALDiskToMaxBytes[walSize]) + promscaleWalSettingsMatrix[memory][walSize][WALBuffersKey] = parse.BytesToPGFormat(walBuffers) + promscaleWalSettingsMatrix[memory][walSize][CheckpointTimeoutKey] = promscaleDefaultCheckpointTimeout + } + } +} + +func TestWALSettingsGroup_GetRecommender(t *testing.T) { + cases := []struct { + profile Profile + recommender string + }{ + {DefaultProfile, "*pgtune.WALRecommender"}, + {PromscaleProfile, "*pgtune.PromscaleWALRecommender"}, + } + + sg := WALSettingsGroup{totalMemory: 1, walDiskSize: 1} + for _, k := range cases { + r := sg.GetRecommender(k.profile) + y := fmt.Sprintf("%T", r) + if y != k.recommender { + t.Errorf("Expected to get a %s using the %s profile but got %s", k.recommender, k.profile, y) } } } @@ -71,16 +113,34 @@ func TestWALRecommenderRecommend(t *testing.T) { } } -func TestWALRecommenderRecommendPanic(t *testing.T) { - func() { - r := NewWALRecommender(0, 0) - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestPromscaleWALRecommenderRecommend(t *testing.T) { + for totalMemory, outerMatrix := range promscaleWalSettingsMatrix { + for walSize, matrix := range outerMatrix { + r := NewPromscaleWALRecommender(totalMemory, walSize) + testRecommender(t, r, WALKeys, matrix) + } + } +} + +func TestPromscaleWALRecommenderCheckpointTimeout(t *testing.T) { + // recommendation for checkpoint timeout should not be impacted by totalMemory or walDiskSize + for i := uint64(0); i < 1000000; i++ { + r := NewPromscaleWALRecommender(i, i) + if v := r.Recommend(CheckpointTimeoutKey); v != promscaleDefaultCheckpointTimeout { + t.Errorf("Expected %s for %s, but got %s", promscaleDefaultCheckpointTimeout, CheckpointTimeoutKey, v) + } + } +} + +func TestWALRecommenderNoRecommendation(t *testing.T) { + r := NewWALRecommender(0, 0) + if r.Recommend("foo") != NoRecommendation { + t.Errorf("Recommendation was provided for %s when there should have been none", "foo") + } + + if r.Recommend(CheckpointTimeoutKey) != NoRecommendation { + t.Errorf("Recommendation was provided for %s when there should have been none", CheckpointTimeoutKey) + } } func TestWALSettingsGroup(t *testing.T) { @@ -90,7 +150,52 @@ func TestWALSettingsGroup(t *testing.T) { config.Memory = totalMemory config.WALDiskSize = walSize sg := GetSettingsGroup(WALLabel, config) - testSettingGroup(t, sg, matrix, WALLabel, WALKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, WALLabel, WALKeys) } } + + for totalMemory, outerMatrix := range promscaleWalSettingsMatrix { + for walSize, matrix := range outerMatrix { + config := getDefaultTestSystemConfig(t) + config.Memory = totalMemory + config.WALDiskSize = walSize + sg := GetSettingsGroup(WALLabel, config) + testSettingGroup(t, sg, PromscaleProfile, matrix, WALLabel, WALKeys) + } + } +} + +func TestWALFloatParserParseFloat(t *testing.T) { + v := &WALFloatParser{} + + s := "8" + parse.GB + want := float64(8 * parse.Gigabyte) + got, err := v.ParseFloat(MaxWALKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } + + s = "1000" + want = 1000.0 + got, err = v.ParseFloat(WALBuffersKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } + + s = "33" + parse.Minutes.String() + conversion, _ := parse.TimeConversion(parse.Minutes, parse.Milliseconds) + want = 33.0 * conversion + got, err = v.ParseFloat(CheckpointTimeoutKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } } diff --git a/pkg/tstune/tune_settings.go b/pkg/tstune/tune_settings.go index 74319ec..a29cab2 100644 --- a/pkg/tstune/tune_settings.go +++ b/pkg/tstune/tune_settings.go @@ -3,10 +3,8 @@ package tstune import ( "fmt" "regexp" - "strconv" "time" - "github.com/timescale/timescaledb-tune/internal/parse" "github.com/timescale/timescaledb-tune/pkg/pgtune" ) @@ -59,35 +57,7 @@ func init() { setup(pgtune.ParallelKeys) setup(pgtune.WALKeys) setup(pgtune.MiscKeys) -} - -type floatParser interface { - ParseFloat(string) (float64, error) -} - -type bytesFloatParser struct{} - -func (v *bytesFloatParser) ParseFloat(s string) (float64, error) { - temp, err := parse.PGFormatToBytes(s) - return float64(temp), err -} - -type numericFloatParser struct{} - -func (v *numericFloatParser) ParseFloat(s string) (float64, error) { - return strconv.ParseFloat(s, 64) -} - -// getFloatParser returns the correct floatParser for a given pgtune.Recommender. -func getFloatParser(r pgtune.Recommender) floatParser { - switch r.(type) { - case *pgtune.MemoryRecommender: - return &bytesFloatParser{} - case *pgtune.WALRecommender: - return &bytesFloatParser{} - default: - return &numericFloatParser{} - } + setup(pgtune.BgwriterKeys) } // keyToRegex takes a conf file key/param name and creates the correct regular diff --git a/pkg/tstune/tune_settings_test.go b/pkg/tstune/tune_settings_test.go index a22c7c8..6ac680c 100644 --- a/pkg/tstune/tune_settings_test.go +++ b/pkg/tstune/tune_settings_test.go @@ -5,9 +5,6 @@ import ( "regexp" "testing" "time" - - "github.com/timescale/timescaledb-tune/internal/parse" - "github.com/timescale/timescaledb-tune/pkg/pgtune" ) // To make the test less flaky, we 0 out the seconds to make the comparison @@ -42,58 +39,6 @@ func TestOurParamToValue(t *testing.T) { _ = ourParamString("not_a_real_param") } -func TestBytesFloatParserParseFloat(t *testing.T) { - s := "8" + parse.GB - want := float64(8 * parse.Gigabyte) - v := &bytesFloatParser{} - got, err := v.ParseFloat(s) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if got != want { - t.Errorf("incorrect result: got %f want %f", got, want) - } -} - -func TestNumericFloatParserParseFloat(t *testing.T) { - s := "8.245" - want := 8.245 - v := &numericFloatParser{} - got, err := v.ParseFloat(s) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if got != want { - t.Errorf("incorrect result: got %f want %f", got, want) - } -} - -func TestGetFloatParser(t *testing.T) { - switch x := (getFloatParser(&pgtune.MemoryRecommender{})).(type) { - case *bytesFloatParser: - default: - t.Errorf("wrong validator type for MemoryRecommender: got %T", x) - } - - switch x := (getFloatParser(&pgtune.WALRecommender{})).(type) { - case *bytesFloatParser: - default: - t.Errorf("wrong validator type for WALRecommender: got %T", x) - } - - switch x := (getFloatParser(&pgtune.ParallelRecommender{})).(type) { - case *numericFloatParser: - default: - t.Errorf("wrong validator type for ParallelRecommender: got %T", x) - } - - switch x := (getFloatParser(&pgtune.MiscRecommender{})).(type) { - case *numericFloatParser: - default: - t.Errorf("wrong validator type for MiscRecommender: got %T", x) - } -} - const ( testKey = "test_setting" testKeyMeta = "test.setting" diff --git a/pkg/tstune/tuner.go b/pkg/tstune/tuner.go index bc10207..c71d88a 100644 --- a/pkg/tstune/tuner.go +++ b/pkg/tstune/tuner.go @@ -83,8 +83,9 @@ type TunerFlags struct { YesAlways bool // always respond yes to prompts Quiet bool // show only the bare necessities UseColor bool // use color in output - DryRun bool // whether to actual persist changes to disk + DryRun bool // whether to actually persist changes to disk Restore bool // whether to restore a backup + Profile string // a specific "mode" to provide recommendations tailored to a special workload type, e.g. "promscale" } // Tuner represents the tuning program for TimescaleDB. @@ -247,6 +248,12 @@ func (t *Tuner) Run(flags *TunerFlags, in io.Reader, out io.Writer, outErr io.Wr } } + profile, err := pgtune.ParseProfile(t.flags.Profile) + ifErrHandle(err) + if profile != pgtune.DefaultProfile { + t.handler.p.Statement("Tuning with profile: %s", profile) + } + // Before proceeding, make sure we have a valid system config config, err := t.initializeSystemConfig() ifErrHandle(err) @@ -290,7 +297,7 @@ func (t *Tuner) Run(flags *TunerFlags, in io.Reader, out io.Writer, outErr io.Wr // Process the tuning of settings if t.flags.Quiet { - err = t.processQuiet(config) + err = t.processQuiet(config, profile) ifErrHandle(err) } else { err = t.processSharedLibLine() @@ -299,7 +306,7 @@ func (t *Tuner) Run(flags *TunerFlags, in io.Reader, out io.Writer, outErr io.Wr fmt.Fprintf(t.handler.outErr, "\n") err = t.promptUntilValidInput(promptTune+promptYesNo, newYesNoChecker("")) if err == nil { - err = t.processTunables(config) + err = t.processTunables(config, profile) ifErrHandle(err) } else if err.Error() != "" { // error msg of "" is response when user selects no to tuning t.handler.errorExit(err) @@ -435,10 +442,10 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar continue } - rv := getFloatParser(recommender) + rv := pgtune.GetFloatParser(recommender) // parse the value already there; if unparseable, should show our rec - curr, err := rv.ParseFloat(r.value) + curr, err := rv.ParseFloat(k, r.value) if err != nil { show[k] = true continue @@ -446,7 +453,11 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar // get and parse our recommendation; fail if for we can't rec := recommender.Recommend(k) - target, err := rv.ParseFloat(rec) + if rec == pgtune.NoRecommendation { + // don't bother adding it to the map. no recommendation + continue + } + target, err := rv.ParseFloat(k, rec) if err != nil { return nil, fmt.Errorf("unexpected parsing problem: %v", err) } @@ -459,7 +470,7 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar return show, nil } -func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { +func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup, profile pgtune.Profile) error { label := sg.Label() quiet := t.flags.Quiet if !quiet { @@ -467,7 +478,7 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { t.handler.p.Statement(fmt.Sprintf("%s%s settings recommendations", strings.ToUpper(label[:1]), label[1:])) } keys := sg.Keys() - recommender := sg.GetRecommender() + recommender := sg.GetRecommender(profile) // Get a map of only the settings that are missing, commented out, or not "close enough" to our recommendation. show, err := checkIfShouldShowSetting(keys, t.cfs.tuneParseResults, recommender) @@ -497,6 +508,11 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { // Display current settings, but only those with new recommendations t.handler.p.Statement(currentLabel) doWithVisibile(func(r *tunableParseResult) { + // don't bother displaying current settings for keys for which we have no recommendation + rec := recommender.Recommend(r.key) + if rec == pgtune.NoRecommendation { + return + } if r.idx == -1 { t.handler.p.Error("missing", r.key) return @@ -513,7 +529,12 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { } // Recommendations are always displayed, but the label above may not be doWithVisibile(func(r *tunableParseResult) { - fmt.Fprintf(t.handler.out, fmtTunableParam+"\n", r.key, recommender.Recommend(r.key), "") // don't print comment, too cluttered + rec := recommender.Recommend(r.key) + // skip keys for which we have no recommendation + if rec == pgtune.NoRecommendation { + return + } + fmt.Fprintf(t.handler.out, fmtTunableParam+"\n", r.key, rec, "") // don't print comment, too cluttered }) // Prompt the user for input (only in non-quiet mode) @@ -531,7 +552,11 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { // If we reach here, it means the user accepted our recommendations, so update the lines doWithVisibile(func(r *tunableParseResult) { - newLine := &configLine{content: fmt.Sprintf(fmtTunableParam, r.key, recommender.Recommend(r.key), r.extra)} // do write comment into file + rec := recommender.Recommend(r.key) + if rec == pgtune.NoRecommendation { + return + } + newLine := &configLine{content: fmt.Sprintf(fmtTunableParam, r.key, rec, r.extra)} // do write comment into file if r.idx == -1 { t.cfs.lines = append(t.cfs.lines, newLine) } else { @@ -547,7 +572,7 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { // processTunables handles user interactions for updating the conf file when it comes // to parameters than be tuned, e.g. memory. -func (t *Tuner) processTunables(config *pgtune.SystemConfig) error { +func (t *Tuner) processTunables(config *pgtune.SystemConfig, profile pgtune.Profile) error { quiet := t.flags.Quiet if !quiet { t.handler.p.Statement(statementTunableIntro, parse.BytesToDecimalFormat(config.Memory), config.CPUs, config.PGMajorVersion) @@ -556,16 +581,17 @@ func (t *Tuner) processTunables(config *pgtune.SystemConfig) error { pgtune.MemoryLabel, pgtune.ParallelLabel, pgtune.WALLabel, + pgtune.BgwriterLabel, pgtune.MiscLabel, } for _, label := range tunables { sg := pgtune.GetSettingsGroup(label, config) - r := sg.GetRecommender() + r := sg.GetRecommender(profile) if !r.IsAvailable() { continue } - err := t.processSettingsGroup(sg) + err := t.processSettingsGroup(sg, profile) if err != nil { return err } @@ -629,7 +655,7 @@ func (w *counterWriter) Write(p []byte) (int, error) { } // processQuiet handles the iteractions when the user wants "quiet" output. -func (t *Tuner) processQuiet(config *pgtune.SystemConfig) error { +func (t *Tuner) processQuiet(config *pgtune.SystemConfig, profile pgtune.Profile) error { t.handler.p.Statement(statementTunableIntro, parse.BytesToDecimalFormat(config.Memory), config.CPUs, config.PGMajorVersion) // Replace the print function with a version that counts how many times it @@ -656,7 +682,7 @@ func (t *Tuner) processQuiet(config *pgtune.SystemConfig) error { } // print out all tunables that need to be changed - err := t.processTunables(config) + err := t.processTunables(config, profile) if err != nil { return err } diff --git a/pkg/tstune/tuner_test.go b/pkg/tstune/tuner_test.go index 7322e69..1c1b8a8 100644 --- a/pkg/tstune/tuner_test.go +++ b/pkg/tstune/tuner_test.go @@ -1060,9 +1060,11 @@ type testSettingsGroup struct { keys []string } -func (sg *testSettingsGroup) Label() string { return "foo" } -func (sg *testSettingsGroup) Keys() []string { return sg.keys } -func (sg *testSettingsGroup) GetRecommender() pgtune.Recommender { return &badRecommender{} } +func (sg *testSettingsGroup) Label() string { return "foo" } +func (sg *testSettingsGroup) Keys() []string { return sg.keys } +func (sg *testSettingsGroup) GetRecommender(profile pgtune.Profile) pgtune.Recommender { + return &badRecommender{} +} func getDefaultSystemConfig(t *testing.T) *pgtune.SystemConfig { config, err := pgtune.NewSystemConfig(testMem, testCPUs, pgutils.MajorVersion10, testWALDisk, testMaxConns, testWorkers) @@ -1077,6 +1079,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { cases := []struct { desc string ts pgtune.SettingsGroup + profile pgtune.Profile lines []string input string wantStatements uint64 @@ -1089,6 +1092,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "bad recommender", ts: &testSettingsGroup{pgtune.ParallelKeys}, + profile: pgtune.DefaultProfile, lines: []string{fmt.Sprintf("%s = 1.0", pgtune.ParallelKeys[0])}, wantStatements: 1, // only intro remark wantPrints: 1, // one for initial newline @@ -1097,6 +1101,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "no keys, no need to prompt", ts: &testSettingsGroup{}, + profile: pgtune.DefaultProfile, lines: memSettingsCorrect, wantStatements: 1, // only intro remark wantPrompts: 0, @@ -1107,6 +1112,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - commented", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsCommented, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1118,6 +1124,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - wrong", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsWrongVal, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1129,6 +1136,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - missing", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsMissing, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1141,6 +1149,19 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - comment+wrong", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, + lines: memSettingsCommentWrong, + input: " \ny\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 2, // first input is blank + wantPrints: 5, // one for initial newline + two settings, displayed twice + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - comment+wrong promscale", + ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.PromscaleProfile, // should produce the same results as the default profile lines: memSettingsCommentWrong, input: " \ny\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1152,6 +1173,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - comment+wrong+missing", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsCommentWrongMissing, input: " \n \ny\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1164,6 +1186,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - all wrong, but skip", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsAllWrong, input: "s\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1176,6 +1199,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - all wrong, but quit", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsAllWrong, input: " \nqUIt\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1187,6 +1211,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - all wrong", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsAllWrong, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1198,6 +1223,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "label capitalized", ts: pgtune.GetSettingsGroup(pgtune.WALLabel, config), + profile: pgtune.DefaultProfile, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label wantPrompts: 1, @@ -1206,12 +1232,38 @@ func TestTunerProcessSettingsGroup(t *testing.T) { successMsg: "WAL settings will be updated", shouldErr: false, }, + { + desc: "wal - checkpoint_timeout promscale", + ts: pgtune.GetSettingsGroup(pgtune.WALLabel, config), + profile: pgtune.PromscaleProfile, + lines: []string{"checkpoint_timeout = 5m"}, + input: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 6, // one for initial newline + 3 for recommendations + wantErrors: 3, // 3 are missing + successMsg: "WAL settings will be updated", + shouldErr: false, + }, + { + desc: "bgwriter", + ts: pgtune.GetSettingsGroup(pgtune.BgwriterLabel, config), + profile: pgtune.PromscaleProfile, + lines: []string{"bgwriter_delay = 13s", "bgwriter_lru_maxpages = 1000"}, + input: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 5, + wantErrors: 0, + successMsg: "background writer settings will be updated", + shouldErr: false, + }, } for _, c := range cases { tuner := newTunerWithDefaultFlagsForInputs(t, c.input, c.lines) - err := tuner.processSettingsGroup(c.ts) + err := tuner.processSettingsGroup(c.ts, c.profile) if err != nil && !c.shouldErr { t.Errorf("%s: unexpected error: %v", c.desc, err) } else if err == nil && c.shouldErr { @@ -1285,10 +1337,11 @@ func TestTunerProcessTunables(t *testing.T) { idx += 3 } checkStmt("Memory settings recommendations") - if wantGroups > 3 { + if wantGroups > 4 { checkStmt("Parallelism settings recommendations") } checkStmt("WAL settings recommendations") + checkStmt("Background writer settings recommendations") checkStmt("Miscellaneous settings recommendations") } input := "y\ny\ny\ny\n" @@ -1297,15 +1350,31 @@ func TestTunerProcessTunables(t *testing.T) { handler := setupDefaultTestIO(input) cfs := &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} tuner := newTunerWithDefaultFlags(handler, cfs) - tuner.processTunables(config) + tuner.processTunables(config, pgtune.DefaultProfile) + check(tuner.handler, config, 5) + + // changes to parallelism settings should not be recommended if only 1 CPU + config.CPUs = 1 + handler = setupDefaultTestIO(input) + cfs = &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} + tuner = newTunerWithDefaultFlags(handler, cfs) + tuner.processTunables(config, pgtune.DefaultProfile) check(tuner.handler, config, 4) + config = getDefaultSystemConfig(t) + handler = setupDefaultTestIO(input) + cfs = &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} + tuner = newTunerWithDefaultFlags(handler, cfs) + tuner.processTunables(config, pgtune.PromscaleProfile) + check(tuner.handler, config, 5) + + // changes to parallelism settings should not be recommended if only 1 CPU config.CPUs = 1 handler = setupDefaultTestIO(input) cfs = &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} tuner = newTunerWithDefaultFlags(handler, cfs) - tuner.processTunables(config) - check(tuner.handler, config, 3) + tuner.processTunables(config, pgtune.PromscaleProfile) + check(tuner.handler, config, 4) } var ( @@ -1460,7 +1529,7 @@ func TestTunerProcessQuiet(t *testing.T) { tuner := newTunerWithDefaultFlagsForInputs(t, input, c.lines) tuner.flags.Quiet = true - err := tuner.processQuiet(config) + err := tuner.processQuiet(config, pgtune.DefaultProfile) if err != nil && !c.shouldErr { t.Errorf("%s: unexpected error: %v", c.desc, err) } else if err == nil && c.shouldErr {