diff --git a/README.md b/README.md index 3fc5c50..1b58d4b 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,14 @@ that you accepted from the prompts. #### Other invocations +By default, timescaledb-tune provides recommendations for a typical timescaledb workload. The `--profile` flag can be +used to tailor the recommendations for other workload types. Currently, the only non-default profile is "promscale". +The `TSTUNE_PROFILE` environment variable can also be used to affect this behavior. + +```bash +$ timescaledb-tune --profile promscale +``` + If you want recommendations for a specific amount of memory and/or CPUs: ```bash $ timescaledb-tune --memory="4GB" --cpus=2 diff --git a/cmd/timescaledb-tune/main.go b/cmd/timescaledb-tune/main.go index b5d37a5..cf0a678 100644 --- a/cmd/timescaledb-tune/main.go +++ b/cmd/timescaledb-tune/main.go @@ -45,9 +45,16 @@ func init() { flag.BoolVar(&f.UseColor, "color", true, "Use color in output (works best on dark terminals)") flag.BoolVar(&f.DryRun, "dry-run", false, "Whether to just show the changes without overwriting the configuration file") flag.BoolVar(&f.Restore, "restore", false, "Whether to restore a previously made conf file backup") + flag.StringVar(&f.Profile, "profile", "", "a specific \"mode\" for tailoring recommendations to a special workload type. If blank or unspecified, a default is used unless the TSTUNE_PROFILE environment variable is set. Valid values: \"promscale\"") flag.BoolVar(&showVersion, "version", false, "Show the version of this tool") flag.Parse() + + // the TSTUNE_PROFILE environment variable overrides the --profile flag if the --profile is blank or unset + // this is designed for docker usage + if val := os.Getenv("TSTUNE_PROFILE"); val != "" && f.Profile == "" { + f.Profile = val + } } func main() { diff --git a/internal/parse/parse.go b/internal/parse/parse.go index 318fbdc..302971b 100644 --- a/internal/parse/parse.go +++ b/internal/parse/parse.go @@ -7,6 +7,8 @@ import ( "math" "regexp" "strconv" + "strings" + "time" ) // Byte equivalents (using 1024) of common byte measurements @@ -19,23 +21,93 @@ const ( // Suffixes for byte measurements that are valid to PostgreSQL const ( - TB = "TB" // terabyte - GB = "GB" // gigabyte - MB = "MB" // megabyte - KB = "kB" // kilobyte - B = "" // no unit, therefore: bytes + TB = "TB" // terabyte + GB = "GB" // gigabyte + MB = "MB" // megabyte + KB = "kB" // kilobyte + B = "" // no unit, therefore: bytes ) +// TimeUnit represents valid suffixes for time measurements used by PostgreSQL settings +// https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values +type TimeUnit int64 + +const ( + Microseconds TimeUnit = iota + Milliseconds + Seconds + Minutes + Hours + Days +) + +func (tu TimeUnit) String() string { + switch tu { + case Microseconds: + return "us" + case Milliseconds: + return "ms" + case Seconds: + return "s" + case Minutes: + return "min" + case Hours: + return "h" + case Days: + return "d" + default: + return "unrecognized" + } +} + +func ParseTimeUnit(val string) (TimeUnit, error) { + switch strings.ToLower(val) { + case "us": + return Microseconds, nil + case "ms": + return Milliseconds, nil + case "s": + return Seconds, nil + case "min": + return Minutes, nil + case "h": + return Hours, nil + case "d": + return Days, nil + default: + return TimeUnit(-1), fmt.Errorf(errUnrecognizedTimeUnitsFmt, val) + } +} + +// VarType represents the values from the vartype column of the pg_settings table +type VarType int64 + +const ( + VarTypeReal VarType = iota + VarTypeInteger +) + +func (v VarType) String() string { + switch v { + case VarTypeReal: + return "real" + case VarTypeInteger: + return "integer" + default: + return "unrecognized" + } +} + const ( - errIncorrectFormatFmt = "incorrect PostgreSQL bytes format: '%s'" - errCouldNotParseBytesFmt = "could not parse bytes number: %v" - errCouldNotParseVersionFmt = "unable to parse PG version string: %s" - errUnknownMajorVersionFmt = "unknown major PG version: %s" + errIncorrectBytesFormatFmt = "incorrect PostgreSQL bytes format: '%s'" + errIncorrectTimeFormatFmt = "incorrect PostgreSQL time format: '%s'" + errCouldNotParseBytesFmt = "could not parse bytes number: %v" + errUnrecognizedTimeUnitsFmt = "unrecognized time units: %s" ) var ( - pgBytesRegex = regexp.MustCompile("^([0-9]+)((?:k|M|G|T)B)?$") - pgVersionRegex = regexp.MustCompile("^PostgreSQL ([0-9]+?).([0-9]+?).*") + pgBytesRegex = regexp.MustCompile("^(?:')?([0-9]+)((?:k|M|G|T)B)?(?:')?$") + pgTimeRegex = regexp.MustCompile(`^(?:')?([0-9]+(\.[0-9]+)?)(?:\s*)(us|ms|s|min|h|d)?(?:')?$`) ) func parseIntToFloatUnits(bytes uint64) (float64, string) { @@ -98,7 +170,7 @@ func BytesToPGFormat(bytes uint64) string { func PGFormatToBytes(val string) (uint64, error) { res := pgBytesRegex.FindStringSubmatch(val) if len(res) != 3 { - return 0.0, fmt.Errorf(errIncorrectFormatFmt, val) + return 0.0, fmt.Errorf(errIncorrectBytesFormatFmt, val) } num, err := strconv.ParseInt(res[1], 10, 64) if err != nil { @@ -121,3 +193,119 @@ func PGFormatToBytes(val string) (uint64, error) { } return ret, nil } + +func NextSmallerTimeUnits(units TimeUnit) TimeUnit { + switch units { + case Microseconds: + return Microseconds + case Milliseconds: + return Microseconds + case Seconds: + return Milliseconds + case Minutes: + return Seconds + case Hours: + return Minutes + default: // Days + return Hours + } +} + +func UnitsToDuration(units TimeUnit) time.Duration { + switch units { + case Microseconds: + return time.Microsecond + case Milliseconds: + return time.Millisecond + case Seconds: + return time.Second + case Minutes: + return time.Minute + case Hours: + return time.Hour + case Days: + return 24 * time.Hour + default: + return time.Nanosecond + } +} + +func TimeConversion(fromUnits, toUnits TimeUnit) (float64, error) { + return float64(UnitsToDuration(fromUnits)) / float64(UnitsToDuration(toUnits)), nil +} + +func PGFormatToTime(val string, defaultUnits TimeUnit, vt VarType) (float64, TimeUnit, error) { + // the default units, whether units were specified, and the variable type ALL impact how the value is interpreted + // https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values + + // select unit, vartype, array_agg(name) from pg_settings where unit in ('us', 'ms', 's', 'm', 'h', 'd') group by 1, 2; + + // parse it + res := pgTimeRegex.FindStringSubmatch(val) + if res == nil || len(res) < 2 { + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) + } + + // extract the numeric portion + v, err := strconv.ParseFloat(res[1], 64) + if err != nil { + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) + } + + // extract the units or use the default + unitsWereUnspecified := true + units := defaultUnits + if len(res) >= 4 && res[3] != "" { + unitsWereUnspecified = false + units, err = ParseTimeUnit(res[3]) + if err != nil { + return -1.0, TimeUnit(-1), err + } + } + + convert := func(v float64, units TimeUnit, vt VarType) (float64, TimeUnit, error) { + if _, fract := math.Modf(v); fract < math.Nextafter(0.0, 1.0) { + // not distinguishable as a fractional value + switch vt { + case VarTypeInteger: + return math.Trunc(v), units, nil + case VarTypeReal: + return v, units, nil + } + } else { + // IS a fractional value. it had a decimal component + toUnits := NextSmallerTimeUnits(units) + if err != nil { + return -1.0, TimeUnit(-1), err + } + conv, err := TimeConversion(units, toUnits) + if err != nil { + return -1.0, TimeUnit(-1), err + } + return math.Round(v * conv), toUnits, nil + } + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) + } + + if unitsWereUnspecified { + switch vt { + case VarTypeInteger: + return math.Round(v), units, nil + case VarTypeReal: + return convert(v, units, vt) + } + } else /* units WERE specified */ { + if units == defaultUnits { + switch vt { + case VarTypeInteger: + return math.Round(v), units, nil + case VarTypeReal: + return convert(v, units, vt) + } + } else /* specified units are different from the default units */ { + return convert(v, units, vt) + } + } + // should never get here! + return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) +} diff --git a/internal/parse/parse.md b/internal/parse/parse.md new file mode 100644 index 0000000..9586eb4 --- /dev/null +++ b/internal/parse/parse.md @@ -0,0 +1,136 @@ + +https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values + +Postgres settings come in one of several variable types: + +* enum +* string +* bool +* integer +* real + +Some postgres settings are measured in time. They have default units, but postgres can interpret the following units if provided: + +* us - microseconds +* ms - milliseconds +* s - seconds +* min - minutes +* h - hours +* d - days + +In postgres 14, time-based settings come in three flavors: + +* real milliseconds +* integer milliseconds +* integer seconds + +How postgres interprets a setting's value depends upon: + +1. whether the setting is a real or integer +2. the default units +3. whether units were specified in the input and whether those units are larger, smaller, or equal to the defaults +4. whether the input was a fractional value + +statement_timeout is an integer setting with default units of milliseconds. + +``` +-- no units +set statement_timeout to '155'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 155ms │ +└───────────────────┘ + +-- default units +set statement_timeout to '100ms'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 100ms │ +└───────────────────┘ + +-- non-default units +set statement_timeout to '100s'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 100s │ +└───────────────────┘ + +-- non-default units with fraction +set statement_timeout to '100.3s'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 100300ms │ +└───────────────────┘ + +-- default units with fraction +set statement_timeout to '100.7ms'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 101ms │ +└───────────────────┘ + +-- no units with fraction +set statement_timeout to '155.3'; show statement_timeout; +┌───────────────────┐ +│ statement_timeout │ +├───────────────────┤ +│ 155ms │ +└───────────────────┘ +``` + +vacuum_cost_delay is a real setting with default units of milliseconds. + +``` +-- no units +set vacuum_cost_delay to '99'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 99ms │ +└───────────────────┘ + +-- default units +set vacuum_cost_delay to '100ms'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 100ms │ +└───────────────────┘ + +-- non-default units +set vacuum_cost_delay to '1000us'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 1ms │ +└───────────────────┘ + +-- non-default units with fraction +set vacuum_cost_delay to '100.3us'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 100.3us │ +└───────────────────┘ + +-- default units with fraction +set vacuum_cost_delay to '50.7ms'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 50700us │ +└───────────────────┘ + +-- no units with fraction +set vacuum_cost_delay to '55.3'; show vacuum_cost_delay; +┌───────────────────┐ +│ vacuum_cost_delay │ +├───────────────────┤ +│ 55300us │ +└───────────────────┘ +``` diff --git a/internal/parse/parse_test.go b/internal/parse/parse_test.go index 3d04953..cc7b4b2 100644 --- a/internal/parse/parse_test.go +++ b/internal/parse/parse_test.go @@ -268,22 +268,22 @@ func TestPGFormatToBytes(t *testing.T) { { desc: "incorrect format #1", input: " 64MB", // no leading spaces - errMsg: fmt.Sprintf(errIncorrectFormatFmt, " 64MB"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, " 64MB"), }, { desc: "incorrect format #2", input: "64b", // bytes not allowed - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "64b"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "64b"), }, { desc: "incorrect format #3", input: "64 GB", // no space between num and units, - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "64 GB"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "64 GB"), }, { desc: "incorrect format #4", input: "-64MB", // negative memory is a no-no - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "-64MB"), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "-64MB"), }, { desc: "incorrect format #5", @@ -293,7 +293,7 @@ func TestPGFormatToBytes(t *testing.T) { { desc: "incorrect format #6", input: "5.5" + MB, // decimal memory is a no-no - errMsg: fmt.Sprintf(errIncorrectFormatFmt, "5.5"+MB), + errMsg: fmt.Sprintf(errIncorrectBytesFormatFmt, "5.5"+MB), }, { desc: "valid bytes", @@ -340,6 +340,11 @@ func TestPGFormatToBytes(t *testing.T) { input: "2048" + TB, want: 2048 * Terabyte, }, + { + desc: "valid megabytes, wrapped in single-quotes", + input: "'64MB'", + want: 64 * Megabyte, + }, } for _, c := range cases { @@ -361,3 +366,494 @@ func TestPGFormatToBytes(t *testing.T) { } } + +func TestPGFormatToTime(t *testing.T) { + cases := []struct { + desc string + input string + defUnits TimeUnit + varType VarType + wantNum float64 + wantUnits TimeUnit + errMsg string + }{ + { + desc: "set statement_timeout to '13ms';", + input: "13ms", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '13ms'; #2", + input: "'13ms'", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 7;", + input: "7", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 7.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 13;", + input: "13", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 13.5;", + input: "13.5", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 14.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to 13.4;", + input: "13.4", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '13.4ms';", + input: "13.4ms", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '13min';", + input: "13min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '13.0min';", + input: "13.0min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 13.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '1.5s';", + input: "1.5s", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 1500.0, + wantUnits: Milliseconds, + }, + { + desc: "set statement_timeout to '1.5min';", + input: "1.5min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 90.0, + wantUnits: Seconds, + }, + { + desc: "set statement_timeout to '1.3h';", + input: "1.3h", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 78.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '42.0 min';", + input: "42.0 min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 42.0, + wantUnits: Minutes, + }, + { + desc: "set statement_timeout to '42.1 min';", + input: "42.1 min", + defUnits: Milliseconds, + varType: VarTypeInteger, + wantNum: 2526.0, + wantUnits: Seconds, + }, + { + desc: "set statement_timeout to 'bob';", + input: "bob", + defUnits: Milliseconds, + varType: VarTypeInteger, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "bob"), + }, + { + desc: "set statement_timeout to '42 bob';", + input: "42 bob", + defUnits: Milliseconds, + varType: VarTypeInteger, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "42 bob"), + }, + { + desc: "set vacuum_cost_delay to 250;", + input: "250", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 250.0, + wantUnits: Milliseconds, + }, + { + desc: "set vacuum_cost_delay to 250.0;", + input: "250.0", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 250.0, + wantUnits: Milliseconds, + }, + { + desc: "set vacuum_cost_delay to 1.3;", + input: "1.3", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 1300.0, + wantUnits: Microseconds, + }, + { + desc: "set vacuum_cost_delay to '1.3ms';", + input: "1.3ms", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 1300.0, + wantUnits: Microseconds, + }, + { + desc: "set vacuum_cost_delay to '1300us';", + input: "1300us", + defUnits: Milliseconds, + varType: VarTypeReal, + wantNum: 1300.0, + wantUnits: Microseconds, + }, + { + desc: "37.1 goats", + input: "37.1 goats", + defUnits: Milliseconds, + varType: VarTypeReal, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "37.1 goats"), + }, + { + desc: "37.42.1min", + input: "37.42.1min", + defUnits: Milliseconds, + varType: VarTypeReal, + errMsg: fmt.Sprintf(errIncorrectTimeFormatFmt, "37.42.1min"), + }, + } + + for _, c := range cases { + v, u, err := PGFormatToTime(c.input, c.defUnits, c.varType) + if len(c.errMsg) > 0 { // failure cases + if err == nil { + t.Errorf("%s: unexpectedly err is nil: want %s", c.desc, c.errMsg) + } else if got := err.Error(); got != c.errMsg { + t.Errorf("%s: unexpected err msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) + } + } else { + if err != nil { + t.Errorf("%s: unexpected err: got %v", c.desc, err) + } + if got := v; got != c.wantNum { + t.Errorf("%s: incorrect num: got %f want %f", c.desc, got, c.wantNum) + } + if got := u; got != c.wantUnits { + t.Errorf("%s: incorrect units: got %s want %s", c.desc, got, c.wantUnits) + } + } + } +} + +func TestTimeConversion(t *testing.T) { + // test cases generated with the following query + /* + with x(unit, const, val) as + ( + values + ('us', 'Microseconds' , interval '1 microsecond'), + ('ms', 'Milliseconds' , interval '1 millisecond'), + ('s', 'Seconds' , interval '1 second'), + ('min', 'Minutes' , interval '1 minute'), + ('h', 'Hours' , interval '1 hour'), + ('d', 'Days' , interval '24 hours') + ) + select string_agg(format + ( + $${ + desc: "%s -> %s", + from: %s, + to: %s, + want: %s / %s, + }$$, + f.unit, + t.unit, + f.const, + t.const, + extract(epoch from f.val), + extract(epoch from t.val) + ), E',\n' order by f.const, t.const) + from x f + cross join x t + ; + */ + + cases := []struct { + desc string + from TimeUnit + to TimeUnit + want float64 + errMsg string + }{ + { + desc: "d -> d", + from: Days, + to: Days, + want: 86400.000000 / 86400.000000, + }, + { + desc: "d -> h", + from: Days, + to: Hours, + want: 86400.000000 / 3600.000000, + }, + { + desc: "d -> us", + from: Days, + to: Microseconds, + want: 86400.000000 / 0.000001, + }, + { + desc: "d -> ms", + from: Days, + to: Milliseconds, + want: 86400.000000 / 0.001000, + }, + { + desc: "d -> min", + from: Days, + to: Minutes, + want: 86400.000000 / 60.000000, + }, + { + desc: "d -> s", + from: Days, + to: Seconds, + want: 86400.000000 / 1.000000, + }, + { + desc: "h -> d", + from: Hours, + to: Days, + want: 3600.000000 / 86400.000000, + }, + { + desc: "h -> h", + from: Hours, + to: Hours, + want: 3600.000000 / 3600.000000, + }, + { + desc: "h -> us", + from: Hours, + to: Microseconds, + want: 3600.000000 / 0.000001, + }, + { + desc: "h -> ms", + from: Hours, + to: Milliseconds, + want: 3600.000000 / 0.001000, + }, + { + desc: "h -> min", + from: Hours, + to: Minutes, + want: 3600.000000 / 60.000000, + }, + { + desc: "h -> s", + from: Hours, + to: Seconds, + want: 3600.000000 / 1.000000, + }, + { + desc: "us -> d", + from: Microseconds, + to: Days, + want: 0.000001 / 86400.000000, + }, + { + desc: "us -> h", + from: Microseconds, + to: Hours, + want: 0.000001 / 3600.000000, + }, + { + desc: "us -> us", + from: Microseconds, + to: Microseconds, + want: 0.000001 / 0.000001, + }, + { + desc: "us -> ms", + from: Microseconds, + to: Milliseconds, + want: 0.000001 / 0.001000, + }, + { + desc: "us -> min", + from: Microseconds, + to: Minutes, + want: 0.000001 / 60.000000, + }, + { + desc: "us -> s", + from: Microseconds, + to: Seconds, + want: 0.000001 / 1.000000, + }, + { + desc: "ms -> d", + from: Milliseconds, + to: Days, + want: 0.001000 / 86400.000000, + }, + { + desc: "ms -> h", + from: Milliseconds, + to: Hours, + want: 0.001000 / 3600.000000, + }, + { + desc: "ms -> us", + from: Milliseconds, + to: Microseconds, + want: 0.001000 / 0.000001, + }, + { + desc: "ms -> ms", + from: Milliseconds, + to: Milliseconds, + want: 0.001000 / 0.001000, + }, + { + desc: "ms -> min", + from: Milliseconds, + to: Minutes, + want: 0.001000 / 60.000000, + }, + { + desc: "ms -> s", + from: Milliseconds, + to: Seconds, + want: 0.001000 / 1.000000, + }, + { + desc: "min -> d", + from: Minutes, + to: Days, + want: 60.000000 / 86400.000000, + }, + { + desc: "min -> h", + from: Minutes, + to: Hours, + want: 60.000000 / 3600.000000, + }, + { + desc: "min -> us", + from: Minutes, + to: Microseconds, + want: 60.000000 / 0.000001, + }, + { + desc: "min -> ms", + from: Minutes, + to: Milliseconds, + want: 60.000000 / 0.001000, + }, + { + desc: "min -> min", + from: Minutes, + to: Minutes, + want: 60.000000 / 60.000000, + }, + { + desc: "min -> s", + from: Minutes, + to: Seconds, + want: 60.000000 / 1.000000, + }, + { + desc: "s -> d", + from: Seconds, + to: Days, + want: 1.000000 / 86400.000000, + }, + { + desc: "s -> h", + from: Seconds, + to: Hours, + want: 1.000000 / 3600.000000, + }, + { + desc: "s -> us", + from: Seconds, + to: Microseconds, + want: 1.000000 / 0.000001, + }, + { + desc: "s -> ms", + from: Seconds, + to: Milliseconds, + want: 1.000000 / 0.001000, + }, + { + desc: "s -> min", + from: Seconds, + to: Minutes, + want: 1.000000 / 60.000000, + }, + { + desc: "s -> s", + from: Seconds, + to: Seconds, + want: 1.000000 / 1.000000, + }, + } + + for _, c := range cases { + conv, err := TimeConversion(c.from, c.to) + if c.errMsg != "" { + if err != nil { + t.Errorf("%s: unexpectedly err is nil: want %s", c.desc, c.errMsg) + } else if got := err.Error(); got != c.errMsg { + t.Errorf("%s: unexpected err msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) + } + } else { + if err != nil { + t.Errorf("%s: unexpected err: got %v", c.desc, err) + } + if got := conv; got != c.want { + t.Errorf("%s: incorrect conv: got %f want %f", c.desc, got, c.want) + } + } + } +} diff --git a/pkg/pgtune/background_writer.go b/pkg/pgtune/background_writer.go new file mode 100644 index 0000000..b18ede5 --- /dev/null +++ b/pkg/pgtune/background_writer.go @@ -0,0 +1,83 @@ +package pgtune + +import "github.com/timescale/timescaledb-tune/internal/parse" + +const ( + BgwriterDelayKey = "bgwriter_delay" + BgwriterLRUMaxPagesKey = "bgwriter_lru_maxpages" + + promscaleDefaultBgwriterDelay = "10ms" + promscaleDefaultBgwriterLRUMaxPages = "100000" +) + +// BgwriterLabel is the label used to refer to the background writer settings group +const BgwriterLabel = "background writer" + +var BgwriterKeys = []string{ + BgwriterDelayKey, + BgwriterLRUMaxPagesKey, +} + +// PromscaleBgwriterRecommender gives recommendations for the background writer for the promscale profile +type PromscaleBgwriterRecommender struct{} + +// IsAvailable returns whether this Recommender is usable given the system resources. Always true. +func (r *PromscaleBgwriterRecommender) IsAvailable() bool { + return true +} + +// Recommend returns the recommended PostgreSQL formatted value for the conf +// file for a given key. +func (r *PromscaleBgwriterRecommender) Recommend(key string) string { + switch key { + case BgwriterDelayKey: + return promscaleDefaultBgwriterDelay + case BgwriterLRUMaxPagesKey: + return promscaleDefaultBgwriterLRUMaxPages + default: + return NoRecommendation + } +} + +// BgwriterSettingsGroup is the SettingsGroup to represent settings that affect the background writer. +type BgwriterSettingsGroup struct { + totalMemory uint64 + cpus int + maxConns uint64 +} + +// Label should always return the value BgwriterLabel. +func (sg *BgwriterSettingsGroup) Label() string { return BgwriterLabel } + +// Keys should always return the BgwriterKeys slice. +func (sg *BgwriterSettingsGroup) Keys() []string { return BgwriterKeys } + +// GetRecommender should return a new Recommender. +func (sg *BgwriterSettingsGroup) GetRecommender(profile Profile) Recommender { + switch profile { + case PromscaleProfile: + return &PromscaleBgwriterRecommender{} + default: + return &NullRecommender{} + } +} + +type BgwriterFloatParser struct{} + +func (v *BgwriterFloatParser) ParseFloat(key string, s string) (float64, error) { + switch key { + case BgwriterDelayKey: + val, units, err := parse.PGFormatToTime(s, parse.Milliseconds, parse.VarTypeInteger) + if err != nil { + return val, err + } + conv, err := parse.TimeConversion(units, parse.Milliseconds) + if err != nil { + return val, err + } + return val * conv, nil + default: + bfp := &numericFloatParser{} + return bfp.ParseFloat(key, s) + } +} diff --git a/pkg/pgtune/background_writer_test.go b/pkg/pgtune/background_writer_test.go new file mode 100644 index 0000000..f5cc0b1 --- /dev/null +++ b/pkg/pgtune/background_writer_test.go @@ -0,0 +1,87 @@ +package pgtune + +import ( + "fmt" + "testing" + + "github.com/timescale/timescaledb-tune/internal/parse" +) + +func TestBgwriterSettingsGroup_GetRecommender(t *testing.T) { + cases := []struct { + profile Profile + recommender string + }{ + {DefaultProfile, "*pgtune.NullRecommender"}, + {PromscaleProfile, "*pgtune.PromscaleBgwriterRecommender"}, + } + + sg := BgwriterSettingsGroup{} + for _, k := range cases { + r := sg.GetRecommender(k.profile) + y := fmt.Sprintf("%T", r) + if y != k.recommender { + t.Errorf("Expected to get a %s using the %s profile but got %s", k.recommender, k.profile, y) + } + } +} + +func TestBgwriterSettingsGroupRecommend(t *testing.T) { + sg := BgwriterSettingsGroup{} + + // the default profile should provide no recommendations + r := sg.GetRecommender(DefaultProfile) + if val := r.Recommend(BgwriterDelayKey); val != NoRecommendation { + t.Errorf("Expected no recommendation for key %s but got %s", BgwriterDelayKey, val) + } + if val := r.Recommend(BgwriterLRUMaxPagesKey); val != NoRecommendation { + t.Errorf("Expected no recommendation for key %s but got %s", BgwriterLRUMaxPagesKey, val) + } + + // the promscale profile should have recommendations + r = sg.GetRecommender(PromscaleProfile) + if val := r.Recommend(BgwriterDelayKey); val != promscaleDefaultBgwriterDelay { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterDelay, BgwriterDelayKey, val) + } + if val := r.Recommend(BgwriterLRUMaxPagesKey); val != promscaleDefaultBgwriterLRUMaxPages { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterLRUMaxPages, BgwriterLRUMaxPagesKey, val) + } +} + +func TestPromscaleBgwriterRecommender(t *testing.T) { + r := PromscaleBgwriterRecommender{} + if !r.IsAvailable() { + t.Error("PromscaleBgwriterRecommender should always be available") + } + if val := r.Recommend(BgwriterDelayKey); val != promscaleDefaultBgwriterDelay { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterDelay, BgwriterDelayKey, val) + } + if val := r.Recommend(BgwriterLRUMaxPagesKey); val != promscaleDefaultBgwriterLRUMaxPages { + t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterLRUMaxPages, BgwriterLRUMaxPagesKey, val) + } +} + +func TestBgwriterFloatParserParseFloat(t *testing.T) { + v := &BgwriterFloatParser{} + + s := "100" + want := 100.0 + got, err := v.ParseFloat(BgwriterLRUMaxPagesKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } + + s = "33" + parse.Minutes.String() + conversion, _ := parse.TimeConversion(parse.Minutes, parse.Milliseconds) + want = 33.0 * conversion + got, err = v.ParseFloat(BgwriterDelayKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } +} diff --git a/pkg/pgtune/float_parser.go b/pkg/pgtune/float_parser.go new file mode 100644 index 0000000..95a70aa --- /dev/null +++ b/pkg/pgtune/float_parser.go @@ -0,0 +1,42 @@ +package pgtune + +import ( + "strconv" + + "github.com/timescale/timescaledb-tune/internal/parse" +) + +type FloatParser interface { + ParseFloat(string, string) (float64, error) +} + +type bytesFloatParser struct{} + +func (v *bytesFloatParser) ParseFloat(key string, s string) (float64, error) { + temp, err := parse.PGFormatToBytes(s) + return float64(temp), err +} + +type numericFloatParser struct{} + +func (v *numericFloatParser) ParseFloat(key string, s string) (float64, error) { + return strconv.ParseFloat(s, 64) +} + +// GetFloatParser returns the correct FloatParser for a given Recommender. +func GetFloatParser(r Recommender) FloatParser { + switch r.(type) { + case *MemoryRecommender: + return &bytesFloatParser{} + case *WALRecommender: + return &WALFloatParser{} + case *PromscaleWALRecommender: + return &WALFloatParser{} + case *PromscaleBgwriterRecommender: + return &BgwriterFloatParser{} + case *ParallelRecommender: + return &numericFloatParser{} + default: + return &numericFloatParser{} + } +} diff --git a/pkg/pgtune/float_parser_test.go b/pkg/pgtune/float_parser_test.go new file mode 100644 index 0000000..390e9cf --- /dev/null +++ b/pkg/pgtune/float_parser_test.go @@ -0,0 +1,77 @@ +package pgtune + +import ( + "testing" + + "github.com/timescale/timescaledb-tune/internal/parse" +) + +func TestBytesFloatParserParseFloat(t *testing.T) { + s := "8" + parse.GB + want := float64(8 * parse.Gigabyte) + v := &bytesFloatParser{} + got, err := v.ParseFloat("foo", s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } +} + +func TestNumericFloatParserParseFloat(t *testing.T) { + s := "8.245" + want := 8.245 + v := &numericFloatParser{} + got, err := v.ParseFloat("foo", s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } +} + +func TestGetFloatParser(t *testing.T) { + switch x := (GetFloatParser(&MemoryRecommender{})).(type) { + case *bytesFloatParser: + default: + t.Errorf("wrong validator type for MemoryRecommender: got %T", x) + } + + switch x := (GetFloatParser(&WALRecommender{})).(type) { + case *WALFloatParser: + default: + t.Errorf("wrong validator type for WALRecommender: got %T", x) + } + + switch x := (GetFloatParser(&PromscaleWALRecommender{})).(type) { + case *WALFloatParser: + default: + t.Errorf("wrong validator type for PromscaleWALRecommender: got %T", x) + } + + switch x := (GetFloatParser(&ParallelRecommender{})).(type) { + case *numericFloatParser: + default: + t.Errorf("wrong validator type for ParallelRecommender: got %T", x) + } + + switch x := (GetFloatParser(&PromscaleBgwriterRecommender{})).(type) { + case *BgwriterFloatParser: + default: + t.Errorf("wrong validator type for PromscaleBgwriterRecommender: got %T", x) + } + + switch x := (GetFloatParser(&MiscRecommender{})).(type) { + case *numericFloatParser: + default: + t.Errorf("wrong validator type for MiscRecommender: got %T", x) + } + + switch x := (GetFloatParser(&NullRecommender{})).(type) { + case *numericFloatParser: + default: + t.Errorf("wrong validator type for NullRecommender: got %T", x) + } +} diff --git a/pkg/pgtune/memory.go b/pkg/pgtune/memory.go index e4125d5..5468963 100644 --- a/pkg/pgtune/memory.go +++ b/pkg/pgtune/memory.go @@ -1,7 +1,6 @@ package pgtune import ( - "fmt" "math" "runtime" @@ -89,7 +88,7 @@ func (r *MemoryRecommender) Recommend(key string) string { } } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -126,6 +125,6 @@ func (sg *MemorySettingsGroup) Label() string { return MemoryLabel } func (sg *MemorySettingsGroup) Keys() []string { return MemoryKeys } // GetRecommender should return a new MemoryRecommender. -func (sg *MemorySettingsGroup) GetRecommender() Recommender { +func (sg *MemorySettingsGroup) GetRecommender(profile Profile) Recommender { return NewMemoryRecommender(sg.totalMemory, sg.cpus, sg.maxConns) } diff --git a/pkg/pgtune/memory_test.go b/pkg/pgtune/memory_test.go index f0e48bd..fa38a7c 100644 --- a/pkg/pgtune/memory_test.go +++ b/pkg/pgtune/memory_test.go @@ -214,16 +214,11 @@ func TestMemoryRecommenderRecommend(t *testing.T) { } } -func TestMemoryRecommenderRecommendPanic(t *testing.T) { - func() { - r := NewMemoryRecommender(1, 1, 1) - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestMemoryRecommenderNoRecommendation(t *testing.T) { + r := NewMemoryRecommender(1, 1, 1) + if r.Recommend("foo") != NoRecommendation { + t.Error("Recommendation was provided when there should have been none") + } } func TestMemorySettingsGroup(t *testing.T) { @@ -236,7 +231,7 @@ func TestMemorySettingsGroup(t *testing.T) { config.maxConns = conns sg := GetSettingsGroup(MemoryLabel, config) - testSettingGroup(t, sg, matrix, MemoryLabel, MemoryKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, MemoryLabel, MemoryKeys) } } } diff --git a/pkg/pgtune/misc.go b/pkg/pgtune/misc.go index 1fceb6e..92fc8b8 100644 --- a/pkg/pgtune/misc.go +++ b/pkg/pgtune/misc.go @@ -141,7 +141,7 @@ func (r *MiscRecommender) Recommend(key string) string { } else if key == EffectiveIOKey { val = getEffectiveIOConcurrency(r.pgMajorVersion) } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -165,6 +165,6 @@ func (sg *MiscSettingsGroup) Keys() []string { } // GetRecommender should return a new MiscRecommender. -func (sg *MiscSettingsGroup) GetRecommender() Recommender { +func (sg *MiscSettingsGroup) GetRecommender(profile Profile) Recommender { return NewMiscRecommender(sg.totalMemory, sg.maxConns, sg.pgMajorVersion) } diff --git a/pkg/pgtune/misc_test.go b/pkg/pgtune/misc_test.go index 1369261..65a9929 100644 --- a/pkg/pgtune/misc_test.go +++ b/pkg/pgtune/misc_test.go @@ -179,16 +179,11 @@ func TestMiscRecommenderRecommend(t *testing.T) { } } -func TestMiscRecommenderRecommendPanic(t *testing.T) { - func() { - r := &MiscRecommender{} - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestMiscRecommenderNoRecommendation(t *testing.T) { + r := &MiscRecommender{} + if r.Recommend("foo") != NoRecommendation { + t.Error("Recommendation was provided when there should have been none") + } } func TestMiscSettingsGroup(t *testing.T) { @@ -200,7 +195,7 @@ func TestMiscSettingsGroup(t *testing.T) { } sg := GetSettingsGroup(MiscLabel, config) - testSettingGroup(t, sg, matrix, MiscLabel, MiscKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, MiscLabel, MiscKeys) } } } diff --git a/pkg/pgtune/null_recommender.go b/pkg/pgtune/null_recommender.go new file mode 100644 index 0000000..cdbad70 --- /dev/null +++ b/pkg/pgtune/null_recommender.go @@ -0,0 +1,16 @@ +package pgtune + +// NullRecommender is a Recommender that returns NoRecommendation for all keys +type NullRecommender struct { +} + +// IsAvailable returns whether this Recommender is usable given the system resources. Always true. +func (r *NullRecommender) IsAvailable() bool { + return true +} + +// Recommend returns the recommended PostgreSQL formatted value for the conf +// file for a given key. +func (r *NullRecommender) Recommend(key string) string { + return NoRecommendation +} diff --git a/pkg/pgtune/null_recommender_test.go b/pkg/pgtune/null_recommender_test.go new file mode 100644 index 0000000..a7764b8 --- /dev/null +++ b/pkg/pgtune/null_recommender_test.go @@ -0,0 +1,17 @@ +package pgtune + +import ( + "fmt" + "testing" +) + +func TestNullRecommender_Recommend(t *testing.T) { + r := &NullRecommender{} + // NullRecommender should ALWAYS return NoRecommendation + for i := 0; i < 1000; i++ { + key := fmt.Sprintf("key%d", i) + if val := r.Recommend(key); val != NoRecommendation { + t.Errorf("Expected no recommendation for key %s but got %s", key, val) + } + } +} diff --git a/pkg/pgtune/parallel.go b/pkg/pgtune/parallel.go index 40f6066..c8057a3 100644 --- a/pkg/pgtune/parallel.go +++ b/pkg/pgtune/parallel.go @@ -69,7 +69,7 @@ func (r *ParallelRecommender) Recommend(key string) string { } else if key == MaxBackgroundWorkers { val = fmt.Sprintf("%d", r.maxBGWorkers) } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -93,6 +93,6 @@ func (sg *ParallelSettingsGroup) Keys() []string { } // GetRecommender should return a new ParallelRecommender. -func (sg *ParallelSettingsGroup) GetRecommender() Recommender { +func (sg *ParallelSettingsGroup) GetRecommender(profile Profile) Recommender { return NewParallelRecommender(sg.cpus, sg.maxBGWorkers) } diff --git a/pkg/pgtune/parallel_test.go b/pkg/pgtune/parallel_test.go index f400572..e14bc2c 100644 --- a/pkg/pgtune/parallel_test.go +++ b/pkg/pgtune/parallel_test.go @@ -99,18 +99,14 @@ func TestParallelRecommenderRecommend(t *testing.T) { } } -func TestParallelRecommenderRecommendPanics(t *testing.T) { - // test invalid key panic - func() { - r := &ParallelRecommender{5, MaxBackgroundWorkersDefault} - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestParallelRecommenderNoRecommendation(t *testing.T) { + r := &ParallelRecommender{5, MaxBackgroundWorkersDefault} + if r.Recommend("foo") != NoRecommendation { + t.Error("Recommendation was provided when there should have been none") + } +} +func TestParallelRecommenderRecommendPanics(t *testing.T) { // test invalid CPU panic func() { defer func() { @@ -146,7 +142,7 @@ func TestParallelSettingsGroup(t *testing.T) { if got := len(sg.Keys()); got != keyCount-1 { t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion96, got, keyCount-1) } - testSettingGroup(t, sg, matrix, ParallelLabel, ParallelKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) // PG10 adds a key config.PGMajorVersion = pgutils.MajorVersion10 @@ -154,14 +150,14 @@ func TestParallelSettingsGroup(t *testing.T) { if got := len(sg.Keys()); got != keyCount { t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion10, got, keyCount) } - testSettingGroup(t, sg, matrix, ParallelLabel, ParallelKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) config.PGMajorVersion = pgutils.MajorVersion11 sg = GetSettingsGroup(ParallelLabel, config) if got := len(sg.Keys()); got != keyCount { t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion11, got, keyCount) } - testSettingGroup(t, sg, matrix, ParallelLabel, ParallelKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) } } diff --git a/pkg/pgtune/tune.go b/pkg/pgtune/tune.go index 0ab1eb2..2e6a6de 100644 --- a/pkg/pgtune/tune.go +++ b/pkg/pgtune/tune.go @@ -3,14 +3,51 @@ // for groups of settings in a PostgreSQL conf file. package pgtune -import "fmt" +import ( + "fmt" + "strings" +) const ( osWindows = "windows" errMaxConnsTooLowFmt = "maxConns must be 0 OR >= %d: got %d" errMaxBGWorkersTooLowFmt = "maxBGWorkers must be >= %d: got %d" + errUnrecognizedProfile = "unrecognized profile: %s" +) + +// Profile is a specific "mode" in which timescaledb-tune can be run to provide recommendations tailored to a +// special workload type, e.g. "promscale" +type Profile int64 + +const ( + DefaultProfile Profile = iota + PromscaleProfile ) +func ParseProfile(s string) (Profile, error) { + switch strings.ToLower(s) { + case "": + return DefaultProfile, nil + case "promscale": + return PromscaleProfile, nil + default: + return DefaultProfile, fmt.Errorf(errUnrecognizedProfile, s) + } +} + +func (p Profile) String() string { + switch p { + case DefaultProfile: + return "" + case PromscaleProfile: + return "promscale" + default: + return "unrecognized" + } +} + +const NoRecommendation = "" + // Recommender is an interface that gives setting recommendations for a given // key, usually grouped by logical settings groups (e.g. MemoryRecommender for memory settings). type Recommender interface { @@ -29,7 +66,7 @@ type SettingsGroup interface { // Keys are the parameter names/keys as they appear in the PostgreSQL conf file, e.g. "shared_buffers". Keys() []string // GetRecommender returns the Recommender that should be used for this group of settings. - GetRecommender() Recommender + GetRecommender(Profile) Recommender } // SystemConfig represents a system's resource configuration, to be used when generating @@ -71,6 +108,8 @@ func GetSettingsGroup(label string, config *SystemConfig) SettingsGroup { return &ParallelSettingsGroup{config.PGMajorVersion, config.CPUs, config.MaxBGWorkers} case label == WALLabel: return &WALSettingsGroup{config.Memory, config.WALDiskSize} + case label == BgwriterLabel: + return &BgwriterSettingsGroup{} case label == MiscLabel: return &MiscSettingsGroup{config.Memory, config.maxConns, config.PGMajorVersion} } diff --git a/pkg/pgtune/tune_test.go b/pkg/pgtune/tune_test.go index 1b649e3..be70d7b 100644 --- a/pkg/pgtune/tune_test.go +++ b/pkg/pgtune/tune_test.go @@ -3,6 +3,7 @@ package pgtune import ( "fmt" "math/rand" + "strings" "testing" ) @@ -80,7 +81,7 @@ func TestNewSystemConfig(t *testing.T) { } func TestGetSettingsGroup(t *testing.T) { - okLabels := []string{MemoryLabel, ParallelLabel, WALLabel, MiscLabel} + okLabels := []string{MemoryLabel, ParallelLabel, WALLabel, BgwriterLabel, MiscLabel} config := getDefaultTestSystemConfig(t) for _, label := range okLabels { sg := GetSettingsGroup(label, config) @@ -109,6 +110,9 @@ func TestGetSettingsGroup(t *testing.T) { if x.walDiskSize != config.WALDiskSize { t.Errorf("WAL group incorrect (wal disk): got %d want %d", x.walDiskSize, config.WALDiskSize) } + case *BgwriterSettingsGroup: + // nothing to check here + continue case *MiscSettingsGroup: if x.totalMemory != config.Memory { t.Errorf("Misc group incorrect (memory): got %d want %d", x.totalMemory, config.Memory) @@ -132,7 +136,7 @@ func TestGetSettingsGroup(t *testing.T) { }() } -func testSettingGroup(t *testing.T, sg SettingsGroup, cases map[string]string, wantLabel string, wantKeys []string) { +func testSettingGroup(t *testing.T, sg SettingsGroup, profile Profile, cases map[string]string, wantLabel string, wantKeys []string) { t.Helper() // No matter how many calls, all calls should return the same @@ -149,7 +153,7 @@ func testSettingGroup(t *testing.T, sg SettingsGroup, cases map[string]string, w } } - r := sg.GetRecommender() + r := sg.GetRecommender(profile) testRecommender(t, r, sg.Keys(), cases) } @@ -168,7 +172,31 @@ func testRecommender(t *testing.T, r Recommender, keys []string, wants map[strin for _, key := range keys { want := wants[key] if got := r.Recommend(key); got != want { - t.Errorf("%v: incorrect result for key %s: got\n%s\nwant\n%s", r, key, got, want) + t.Errorf("%T: incorrect result for key %s: got\n%s\nwant\n%s", r, key, got, want) } } } + +func TestParseProfile(t *testing.T) { + cases := []struct { + input string + expected Profile + }{ + {input: DefaultProfile.String(), expected: DefaultProfile}, + {input: PromscaleProfile.String(), expected: PromscaleProfile}, + {input: strings.ToUpper(PromscaleProfile.String()), expected: PromscaleProfile}, + } + for _, kase := range cases { + actual, err := ParseProfile(kase.input) + if err != nil { + t.Errorf("expected %v for input %s but got an error: %v", kase.expected, kase.input, err) + } + if actual != kase.expected { + t.Errorf("expected %v for input %s but got %v", kase.expected, kase.input, actual) + } + } + + if actual, err := ParseProfile("garbage"); err == nil { + t.Errorf("expected to get an error for unrecognized input, but did not. got %v", actual) + } +} diff --git a/pkg/pgtune/wal.go b/pkg/pgtune/wal.go index 450e66e..d9ea7e4 100644 --- a/pkg/pgtune/wal.go +++ b/pkg/pgtune/wal.go @@ -1,21 +1,22 @@ package pgtune import ( - "fmt" - "github.com/timescale/timescaledb-tune/internal/parse" ) // Keys in the conf file that are tuned related to the WAL const ( - WALBuffersKey = "wal_buffers" - MinWALKey = "min_wal_size" - MaxWALKey = "max_wal_size" - - walMaxDiskPct = 60 // max_wal_size should be 60% of the WAL disk - walBuffersThreshold = 2 * parse.Gigabyte - walBuffersDefault = 16 * parse.Megabyte - defaultMaxWALBytes = 1 * parse.Gigabyte + WALBuffersKey = "wal_buffers" + MinWALKey = "min_wal_size" + MaxWALKey = "max_wal_size" + CheckpointTimeoutKey = "checkpoint_timeout" + + walMaxDiskPct = 60 // max_wal_size should be 60% of the WAL disk + walBuffersThreshold = 2 * parse.Gigabyte + walBuffersDefault = 16 * parse.Megabyte + defaultMaxWALBytes = 1 * parse.Gigabyte + promscaleDefaultMaxWALBytes = 4 * parse.Gigabyte + promscaleDefaultCheckpointTimeout = "900" // 15 minutes expressed in seconds ) // WALLabel is the label used to refer to the WAL settings group @@ -26,6 +27,7 @@ var WALKeys = []string{ WALBuffersKey, MinWALKey, MaxWALKey, + CheckpointTimeoutKey, } // WALRecommender gives recommendations for WALKeys based on system resources @@ -66,7 +68,7 @@ func (r *WALRecommender) Recommend(key string) string { temp := r.calcMaxWALBytes() val = parse.BytesToPGFormat(temp) } else { - panic(fmt.Sprintf("unknown key: %s", key)) + val = NoRecommendation } return val } @@ -102,6 +104,87 @@ func (sg *WALSettingsGroup) Label() string { return WALLabel } func (sg *WALSettingsGroup) Keys() []string { return WALKeys } // GetRecommender should return a new WALRecommender. -func (sg *WALSettingsGroup) GetRecommender() Recommender { - return NewWALRecommender(sg.totalMemory, sg.walDiskSize) +func (sg *WALSettingsGroup) GetRecommender(profile Profile) Recommender { + switch profile { + case PromscaleProfile: + return NewPromscaleWALRecommender(sg.totalMemory, sg.walDiskSize) + default: + return NewWALRecommender(sg.totalMemory, sg.walDiskSize) + } +} + +// PromscaleWALRecommender gives recommendations for WALKeys based on system resources +type PromscaleWALRecommender struct { + WALRecommender +} + +// NewPromscaleWALRecommender returns a PromscaleWALRecommender that recommends based on the given +// totalMemory bytes. +func NewPromscaleWALRecommender(totalMemory, walDiskSize uint64) *PromscaleWALRecommender { + return &PromscaleWALRecommender{ + WALRecommender: WALRecommender{ + totalMemory: totalMemory, + walDiskSize: walDiskSize, + }, + } +} + +// IsAvailable returns whether this Recommender is usable given the system resources. Always true. +func (r *PromscaleWALRecommender) IsAvailable() bool { + return true +} + +// Recommend returns the recommended PostgreSQL formatted value for the conf +// file for a given key. +func (r *PromscaleWALRecommender) Recommend(key string) string { + switch key { + case MinWALKey: + temp := r.promscaleCalcMaxWALBytes() / 2 + return parse.BytesToPGFormat(temp) + case MaxWALKey: + temp := r.promscaleCalcMaxWALBytes() + return parse.BytesToPGFormat(temp) + case CheckpointTimeoutKey: + return promscaleDefaultCheckpointTimeout + default: + return r.WALRecommender.Recommend(key) + } +} + +func (r *WALRecommender) promscaleCalcMaxWALBytes() uint64 { + // If disk size is not given, just use default + if r.walDiskSize == 0 { + return promscaleDefaultMaxWALBytes + } + + // With size given, we want to take up at most walMaxDiskPct, to give + // additional room for safety. + max := uint64(r.walDiskSize*walMaxDiskPct) / 100 + + // WAL segments are 16MB, so it doesn't make sense not to round + // up to the nearest 16MB boundary. + if max%(16*parse.Megabyte) != 0 { + max = (max/(16*parse.Megabyte) + 1) * 16 * parse.Megabyte + } + return max +} + +type WALFloatParser struct{} + +func (v *WALFloatParser) ParseFloat(key string, s string) (float64, error) { + switch key { + case CheckpointTimeoutKey: + val, units, err := parse.PGFormatToTime(s, parse.Milliseconds, parse.VarTypeInteger) + if err != nil { + return val, err + } + conv, err := parse.TimeConversion(units, parse.Milliseconds) + if err != nil { + return val, err + } + return val * conv, nil + default: + bfp := &bytesFloatParser{} + return bfp.ParseFloat(key, s) + } } diff --git a/pkg/pgtune/wal_test.go b/pkg/pgtune/wal_test.go index 139d5cb..ee98aec 100644 --- a/pkg/pgtune/wal_test.go +++ b/pkg/pgtune/wal_test.go @@ -1,6 +1,7 @@ package pgtune import ( + "fmt" "math/rand" "testing" @@ -29,10 +30,20 @@ var walDiskToMaxBytes = map[uint64]uint64{ walDiskDivideEvenly: 5280 * parse.Megabyte, } +var promscaleWALDiskToMaxBytes = map[uint64]uint64{ + walDiskUnset: promscaleDefaultMaxWALBytes, + walDiskDivideUnevenly: 4928 * parse.Megabyte, // nearest 16MB segment + walDiskDivideEvenly: 5280 * parse.Megabyte, +} + // walSettingsMatrix stores the test cases for WALRecommender along with the // expected values for WAL keys. var walSettingsMatrix = map[uint64]map[uint64]map[string]string{} +// walSettingsMatrix stores the test cases for WALRecommender along with the +// expected values for WAL keys. +var promscaleWalSettingsMatrix = map[uint64]map[uint64]map[string]string{} + func init() { for memory, walBuffers := range memoryToWALBuffers { walSettingsMatrix[memory] = make(map[uint64]map[string]string) @@ -41,6 +52,37 @@ func init() { walSettingsMatrix[memory][walSize][MinWALKey] = parse.BytesToPGFormat(walDiskToMaxBytes[walSize] / 2) walSettingsMatrix[memory][walSize][MaxWALKey] = parse.BytesToPGFormat(walDiskToMaxBytes[walSize]) walSettingsMatrix[memory][walSize][WALBuffersKey] = parse.BytesToPGFormat(walBuffers) + walSettingsMatrix[memory][walSize][CheckpointTimeoutKey] = NoRecommendation + } + } + + for memory, walBuffers := range memoryToWALBuffers { + promscaleWalSettingsMatrix[memory] = make(map[uint64]map[string]string) + for walSize := range walDiskToMaxBytes { + promscaleWalSettingsMatrix[memory][walSize] = make(map[string]string) + promscaleWalSettingsMatrix[memory][walSize][MinWALKey] = parse.BytesToPGFormat(promscaleWALDiskToMaxBytes[walSize] / 2) + promscaleWalSettingsMatrix[memory][walSize][MaxWALKey] = parse.BytesToPGFormat(promscaleWALDiskToMaxBytes[walSize]) + promscaleWalSettingsMatrix[memory][walSize][WALBuffersKey] = parse.BytesToPGFormat(walBuffers) + promscaleWalSettingsMatrix[memory][walSize][CheckpointTimeoutKey] = promscaleDefaultCheckpointTimeout + } + } +} + +func TestWALSettingsGroup_GetRecommender(t *testing.T) { + cases := []struct { + profile Profile + recommender string + }{ + {DefaultProfile, "*pgtune.WALRecommender"}, + {PromscaleProfile, "*pgtune.PromscaleWALRecommender"}, + } + + sg := WALSettingsGroup{totalMemory: 1, walDiskSize: 1} + for _, k := range cases { + r := sg.GetRecommender(k.profile) + y := fmt.Sprintf("%T", r) + if y != k.recommender { + t.Errorf("Expected to get a %s using the %s profile but got %s", k.recommender, k.profile, y) } } } @@ -71,16 +113,34 @@ func TestWALRecommenderRecommend(t *testing.T) { } } -func TestWALRecommenderRecommendPanic(t *testing.T) { - func() { - r := NewWALRecommender(0, 0) - defer func() { - if re := recover(); re == nil { - t.Errorf("did not panic when should") - } - }() - r.Recommend("foo") - }() +func TestPromscaleWALRecommenderRecommend(t *testing.T) { + for totalMemory, outerMatrix := range promscaleWalSettingsMatrix { + for walSize, matrix := range outerMatrix { + r := NewPromscaleWALRecommender(totalMemory, walSize) + testRecommender(t, r, WALKeys, matrix) + } + } +} + +func TestPromscaleWALRecommenderCheckpointTimeout(t *testing.T) { + // recommendation for checkpoint timeout should not be impacted by totalMemory or walDiskSize + for i := uint64(0); i < 1000000; i++ { + r := NewPromscaleWALRecommender(i, i) + if v := r.Recommend(CheckpointTimeoutKey); v != promscaleDefaultCheckpointTimeout { + t.Errorf("Expected %s for %s, but got %s", promscaleDefaultCheckpointTimeout, CheckpointTimeoutKey, v) + } + } +} + +func TestWALRecommenderNoRecommendation(t *testing.T) { + r := NewWALRecommender(0, 0) + if r.Recommend("foo") != NoRecommendation { + t.Errorf("Recommendation was provided for %s when there should have been none", "foo") + } + + if r.Recommend(CheckpointTimeoutKey) != NoRecommendation { + t.Errorf("Recommendation was provided for %s when there should have been none", CheckpointTimeoutKey) + } } func TestWALSettingsGroup(t *testing.T) { @@ -90,7 +150,52 @@ func TestWALSettingsGroup(t *testing.T) { config.Memory = totalMemory config.WALDiskSize = walSize sg := GetSettingsGroup(WALLabel, config) - testSettingGroup(t, sg, matrix, WALLabel, WALKeys) + testSettingGroup(t, sg, DefaultProfile, matrix, WALLabel, WALKeys) } } + + for totalMemory, outerMatrix := range promscaleWalSettingsMatrix { + for walSize, matrix := range outerMatrix { + config := getDefaultTestSystemConfig(t) + config.Memory = totalMemory + config.WALDiskSize = walSize + sg := GetSettingsGroup(WALLabel, config) + testSettingGroup(t, sg, PromscaleProfile, matrix, WALLabel, WALKeys) + } + } +} + +func TestWALFloatParserParseFloat(t *testing.T) { + v := &WALFloatParser{} + + s := "8" + parse.GB + want := float64(8 * parse.Gigabyte) + got, err := v.ParseFloat(MaxWALKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } + + s = "1000" + want = 1000.0 + got, err = v.ParseFloat(WALBuffersKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } + + s = "33" + parse.Minutes.String() + conversion, _ := parse.TimeConversion(parse.Minutes, parse.Milliseconds) + want = 33.0 * conversion + got, err = v.ParseFloat(CheckpointTimeoutKey, s) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != want { + t.Errorf("incorrect result: got %f want %f", got, want) + } } diff --git a/pkg/tstune/tune_settings.go b/pkg/tstune/tune_settings.go index 74319ec..a29cab2 100644 --- a/pkg/tstune/tune_settings.go +++ b/pkg/tstune/tune_settings.go @@ -3,10 +3,8 @@ package tstune import ( "fmt" "regexp" - "strconv" "time" - "github.com/timescale/timescaledb-tune/internal/parse" "github.com/timescale/timescaledb-tune/pkg/pgtune" ) @@ -59,35 +57,7 @@ func init() { setup(pgtune.ParallelKeys) setup(pgtune.WALKeys) setup(pgtune.MiscKeys) -} - -type floatParser interface { - ParseFloat(string) (float64, error) -} - -type bytesFloatParser struct{} - -func (v *bytesFloatParser) ParseFloat(s string) (float64, error) { - temp, err := parse.PGFormatToBytes(s) - return float64(temp), err -} - -type numericFloatParser struct{} - -func (v *numericFloatParser) ParseFloat(s string) (float64, error) { - return strconv.ParseFloat(s, 64) -} - -// getFloatParser returns the correct floatParser for a given pgtune.Recommender. -func getFloatParser(r pgtune.Recommender) floatParser { - switch r.(type) { - case *pgtune.MemoryRecommender: - return &bytesFloatParser{} - case *pgtune.WALRecommender: - return &bytesFloatParser{} - default: - return &numericFloatParser{} - } + setup(pgtune.BgwriterKeys) } // keyToRegex takes a conf file key/param name and creates the correct regular diff --git a/pkg/tstune/tune_settings_test.go b/pkg/tstune/tune_settings_test.go index a22c7c8..6ac680c 100644 --- a/pkg/tstune/tune_settings_test.go +++ b/pkg/tstune/tune_settings_test.go @@ -5,9 +5,6 @@ import ( "regexp" "testing" "time" - - "github.com/timescale/timescaledb-tune/internal/parse" - "github.com/timescale/timescaledb-tune/pkg/pgtune" ) // To make the test less flaky, we 0 out the seconds to make the comparison @@ -42,58 +39,6 @@ func TestOurParamToValue(t *testing.T) { _ = ourParamString("not_a_real_param") } -func TestBytesFloatParserParseFloat(t *testing.T) { - s := "8" + parse.GB - want := float64(8 * parse.Gigabyte) - v := &bytesFloatParser{} - got, err := v.ParseFloat(s) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if got != want { - t.Errorf("incorrect result: got %f want %f", got, want) - } -} - -func TestNumericFloatParserParseFloat(t *testing.T) { - s := "8.245" - want := 8.245 - v := &numericFloatParser{} - got, err := v.ParseFloat(s) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if got != want { - t.Errorf("incorrect result: got %f want %f", got, want) - } -} - -func TestGetFloatParser(t *testing.T) { - switch x := (getFloatParser(&pgtune.MemoryRecommender{})).(type) { - case *bytesFloatParser: - default: - t.Errorf("wrong validator type for MemoryRecommender: got %T", x) - } - - switch x := (getFloatParser(&pgtune.WALRecommender{})).(type) { - case *bytesFloatParser: - default: - t.Errorf("wrong validator type for WALRecommender: got %T", x) - } - - switch x := (getFloatParser(&pgtune.ParallelRecommender{})).(type) { - case *numericFloatParser: - default: - t.Errorf("wrong validator type for ParallelRecommender: got %T", x) - } - - switch x := (getFloatParser(&pgtune.MiscRecommender{})).(type) { - case *numericFloatParser: - default: - t.Errorf("wrong validator type for MiscRecommender: got %T", x) - } -} - const ( testKey = "test_setting" testKeyMeta = "test.setting" diff --git a/pkg/tstune/tuner.go b/pkg/tstune/tuner.go index bc10207..c71d88a 100644 --- a/pkg/tstune/tuner.go +++ b/pkg/tstune/tuner.go @@ -83,8 +83,9 @@ type TunerFlags struct { YesAlways bool // always respond yes to prompts Quiet bool // show only the bare necessities UseColor bool // use color in output - DryRun bool // whether to actual persist changes to disk + DryRun bool // whether to actually persist changes to disk Restore bool // whether to restore a backup + Profile string // a specific "mode" to provide recommendations tailored to a special workload type, e.g. "promscale" } // Tuner represents the tuning program for TimescaleDB. @@ -247,6 +248,12 @@ func (t *Tuner) Run(flags *TunerFlags, in io.Reader, out io.Writer, outErr io.Wr } } + profile, err := pgtune.ParseProfile(t.flags.Profile) + ifErrHandle(err) + if profile != pgtune.DefaultProfile { + t.handler.p.Statement("Tuning with profile: %s", profile) + } + // Before proceeding, make sure we have a valid system config config, err := t.initializeSystemConfig() ifErrHandle(err) @@ -290,7 +297,7 @@ func (t *Tuner) Run(flags *TunerFlags, in io.Reader, out io.Writer, outErr io.Wr // Process the tuning of settings if t.flags.Quiet { - err = t.processQuiet(config) + err = t.processQuiet(config, profile) ifErrHandle(err) } else { err = t.processSharedLibLine() @@ -299,7 +306,7 @@ func (t *Tuner) Run(flags *TunerFlags, in io.Reader, out io.Writer, outErr io.Wr fmt.Fprintf(t.handler.outErr, "\n") err = t.promptUntilValidInput(promptTune+promptYesNo, newYesNoChecker("")) if err == nil { - err = t.processTunables(config) + err = t.processTunables(config, profile) ifErrHandle(err) } else if err.Error() != "" { // error msg of "" is response when user selects no to tuning t.handler.errorExit(err) @@ -435,10 +442,10 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar continue } - rv := getFloatParser(recommender) + rv := pgtune.GetFloatParser(recommender) // parse the value already there; if unparseable, should show our rec - curr, err := rv.ParseFloat(r.value) + curr, err := rv.ParseFloat(k, r.value) if err != nil { show[k] = true continue @@ -446,7 +453,11 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar // get and parse our recommendation; fail if for we can't rec := recommender.Recommend(k) - target, err := rv.ParseFloat(rec) + if rec == pgtune.NoRecommendation { + // don't bother adding it to the map. no recommendation + continue + } + target, err := rv.ParseFloat(k, rec) if err != nil { return nil, fmt.Errorf("unexpected parsing problem: %v", err) } @@ -459,7 +470,7 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar return show, nil } -func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { +func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup, profile pgtune.Profile) error { label := sg.Label() quiet := t.flags.Quiet if !quiet { @@ -467,7 +478,7 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { t.handler.p.Statement(fmt.Sprintf("%s%s settings recommendations", strings.ToUpper(label[:1]), label[1:])) } keys := sg.Keys() - recommender := sg.GetRecommender() + recommender := sg.GetRecommender(profile) // Get a map of only the settings that are missing, commented out, or not "close enough" to our recommendation. show, err := checkIfShouldShowSetting(keys, t.cfs.tuneParseResults, recommender) @@ -497,6 +508,11 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { // Display current settings, but only those with new recommendations t.handler.p.Statement(currentLabel) doWithVisibile(func(r *tunableParseResult) { + // don't bother displaying current settings for keys for which we have no recommendation + rec := recommender.Recommend(r.key) + if rec == pgtune.NoRecommendation { + return + } if r.idx == -1 { t.handler.p.Error("missing", r.key) return @@ -513,7 +529,12 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { } // Recommendations are always displayed, but the label above may not be doWithVisibile(func(r *tunableParseResult) { - fmt.Fprintf(t.handler.out, fmtTunableParam+"\n", r.key, recommender.Recommend(r.key), "") // don't print comment, too cluttered + rec := recommender.Recommend(r.key) + // skip keys for which we have no recommendation + if rec == pgtune.NoRecommendation { + return + } + fmt.Fprintf(t.handler.out, fmtTunableParam+"\n", r.key, rec, "") // don't print comment, too cluttered }) // Prompt the user for input (only in non-quiet mode) @@ -531,7 +552,11 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { // If we reach here, it means the user accepted our recommendations, so update the lines doWithVisibile(func(r *tunableParseResult) { - newLine := &configLine{content: fmt.Sprintf(fmtTunableParam, r.key, recommender.Recommend(r.key), r.extra)} // do write comment into file + rec := recommender.Recommend(r.key) + if rec == pgtune.NoRecommendation { + return + } + newLine := &configLine{content: fmt.Sprintf(fmtTunableParam, r.key, rec, r.extra)} // do write comment into file if r.idx == -1 { t.cfs.lines = append(t.cfs.lines, newLine) } else { @@ -547,7 +572,7 @@ func (t *Tuner) processSettingsGroup(sg pgtune.SettingsGroup) error { // processTunables handles user interactions for updating the conf file when it comes // to parameters than be tuned, e.g. memory. -func (t *Tuner) processTunables(config *pgtune.SystemConfig) error { +func (t *Tuner) processTunables(config *pgtune.SystemConfig, profile pgtune.Profile) error { quiet := t.flags.Quiet if !quiet { t.handler.p.Statement(statementTunableIntro, parse.BytesToDecimalFormat(config.Memory), config.CPUs, config.PGMajorVersion) @@ -556,16 +581,17 @@ func (t *Tuner) processTunables(config *pgtune.SystemConfig) error { pgtune.MemoryLabel, pgtune.ParallelLabel, pgtune.WALLabel, + pgtune.BgwriterLabel, pgtune.MiscLabel, } for _, label := range tunables { sg := pgtune.GetSettingsGroup(label, config) - r := sg.GetRecommender() + r := sg.GetRecommender(profile) if !r.IsAvailable() { continue } - err := t.processSettingsGroup(sg) + err := t.processSettingsGroup(sg, profile) if err != nil { return err } @@ -629,7 +655,7 @@ func (w *counterWriter) Write(p []byte) (int, error) { } // processQuiet handles the iteractions when the user wants "quiet" output. -func (t *Tuner) processQuiet(config *pgtune.SystemConfig) error { +func (t *Tuner) processQuiet(config *pgtune.SystemConfig, profile pgtune.Profile) error { t.handler.p.Statement(statementTunableIntro, parse.BytesToDecimalFormat(config.Memory), config.CPUs, config.PGMajorVersion) // Replace the print function with a version that counts how many times it @@ -656,7 +682,7 @@ func (t *Tuner) processQuiet(config *pgtune.SystemConfig) error { } // print out all tunables that need to be changed - err := t.processTunables(config) + err := t.processTunables(config, profile) if err != nil { return err } diff --git a/pkg/tstune/tuner_test.go b/pkg/tstune/tuner_test.go index 7322e69..1c1b8a8 100644 --- a/pkg/tstune/tuner_test.go +++ b/pkg/tstune/tuner_test.go @@ -1060,9 +1060,11 @@ type testSettingsGroup struct { keys []string } -func (sg *testSettingsGroup) Label() string { return "foo" } -func (sg *testSettingsGroup) Keys() []string { return sg.keys } -func (sg *testSettingsGroup) GetRecommender() pgtune.Recommender { return &badRecommender{} } +func (sg *testSettingsGroup) Label() string { return "foo" } +func (sg *testSettingsGroup) Keys() []string { return sg.keys } +func (sg *testSettingsGroup) GetRecommender(profile pgtune.Profile) pgtune.Recommender { + return &badRecommender{} +} func getDefaultSystemConfig(t *testing.T) *pgtune.SystemConfig { config, err := pgtune.NewSystemConfig(testMem, testCPUs, pgutils.MajorVersion10, testWALDisk, testMaxConns, testWorkers) @@ -1077,6 +1079,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { cases := []struct { desc string ts pgtune.SettingsGroup + profile pgtune.Profile lines []string input string wantStatements uint64 @@ -1089,6 +1092,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "bad recommender", ts: &testSettingsGroup{pgtune.ParallelKeys}, + profile: pgtune.DefaultProfile, lines: []string{fmt.Sprintf("%s = 1.0", pgtune.ParallelKeys[0])}, wantStatements: 1, // only intro remark wantPrints: 1, // one for initial newline @@ -1097,6 +1101,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "no keys, no need to prompt", ts: &testSettingsGroup{}, + profile: pgtune.DefaultProfile, lines: memSettingsCorrect, wantStatements: 1, // only intro remark wantPrompts: 0, @@ -1107,6 +1112,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - commented", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsCommented, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1118,6 +1124,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - wrong", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsWrongVal, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1129,6 +1136,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - missing", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsMissing, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1141,6 +1149,19 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - comment+wrong", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, + lines: memSettingsCommentWrong, + input: " \ny\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 2, // first input is blank + wantPrints: 5, // one for initial newline + two settings, displayed twice + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - comment+wrong promscale", + ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.PromscaleProfile, // should produce the same results as the default profile lines: memSettingsCommentWrong, input: " \ny\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1152,6 +1173,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - comment+wrong+missing", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsCommentWrongMissing, input: " \n \ny\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1164,6 +1186,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - all wrong, but skip", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsAllWrong, input: "s\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1176,6 +1199,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - all wrong, but quit", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsAllWrong, input: " \nqUIt\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1187,6 +1211,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "memory - all wrong", ts: pgtune.GetSettingsGroup(pgtune.MemoryLabel, config), + profile: pgtune.DefaultProfile, lines: memSettingsAllWrong, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label @@ -1198,6 +1223,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { { desc: "label capitalized", ts: pgtune.GetSettingsGroup(pgtune.WALLabel, config), + profile: pgtune.DefaultProfile, input: "y\n", wantStatements: 3, // intro remark + current label + recommend label wantPrompts: 1, @@ -1206,12 +1232,38 @@ func TestTunerProcessSettingsGroup(t *testing.T) { successMsg: "WAL settings will be updated", shouldErr: false, }, + { + desc: "wal - checkpoint_timeout promscale", + ts: pgtune.GetSettingsGroup(pgtune.WALLabel, config), + profile: pgtune.PromscaleProfile, + lines: []string{"checkpoint_timeout = 5m"}, + input: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 6, // one for initial newline + 3 for recommendations + wantErrors: 3, // 3 are missing + successMsg: "WAL settings will be updated", + shouldErr: false, + }, + { + desc: "bgwriter", + ts: pgtune.GetSettingsGroup(pgtune.BgwriterLabel, config), + profile: pgtune.PromscaleProfile, + lines: []string{"bgwriter_delay = 13s", "bgwriter_lru_maxpages = 1000"}, + input: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 5, + wantErrors: 0, + successMsg: "background writer settings will be updated", + shouldErr: false, + }, } for _, c := range cases { tuner := newTunerWithDefaultFlagsForInputs(t, c.input, c.lines) - err := tuner.processSettingsGroup(c.ts) + err := tuner.processSettingsGroup(c.ts, c.profile) if err != nil && !c.shouldErr { t.Errorf("%s: unexpected error: %v", c.desc, err) } else if err == nil && c.shouldErr { @@ -1285,10 +1337,11 @@ func TestTunerProcessTunables(t *testing.T) { idx += 3 } checkStmt("Memory settings recommendations") - if wantGroups > 3 { + if wantGroups > 4 { checkStmt("Parallelism settings recommendations") } checkStmt("WAL settings recommendations") + checkStmt("Background writer settings recommendations") checkStmt("Miscellaneous settings recommendations") } input := "y\ny\ny\ny\n" @@ -1297,15 +1350,31 @@ func TestTunerProcessTunables(t *testing.T) { handler := setupDefaultTestIO(input) cfs := &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} tuner := newTunerWithDefaultFlags(handler, cfs) - tuner.processTunables(config) + tuner.processTunables(config, pgtune.DefaultProfile) + check(tuner.handler, config, 5) + + // changes to parallelism settings should not be recommended if only 1 CPU + config.CPUs = 1 + handler = setupDefaultTestIO(input) + cfs = &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} + tuner = newTunerWithDefaultFlags(handler, cfs) + tuner.processTunables(config, pgtune.DefaultProfile) check(tuner.handler, config, 4) + config = getDefaultSystemConfig(t) + handler = setupDefaultTestIO(input) + cfs = &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} + tuner = newTunerWithDefaultFlags(handler, cfs) + tuner.processTunables(config, pgtune.PromscaleProfile) + check(tuner.handler, config, 5) + + // changes to parallelism settings should not be recommended if only 1 CPU config.CPUs = 1 handler = setupDefaultTestIO(input) cfs = &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} tuner = newTunerWithDefaultFlags(handler, cfs) - tuner.processTunables(config) - check(tuner.handler, config, 3) + tuner.processTunables(config, pgtune.PromscaleProfile) + check(tuner.handler, config, 4) } var ( @@ -1460,7 +1529,7 @@ func TestTunerProcessQuiet(t *testing.T) { tuner := newTunerWithDefaultFlagsForInputs(t, input, c.lines) tuner.flags.Quiet = true - err := tuner.processQuiet(config) + err := tuner.processQuiet(config, pgtune.DefaultProfile) if err != nil && !c.shouldErr { t.Errorf("%s: unexpected error: %v", c.desc, err) } else if err == nil && c.shouldErr {