make the tool more abstract

wasp/benchspy/basic.go

@@ -0,0 +1,107 @@
+package benchspy
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/chainlink-testing-framework/wasp"
+)
+
+// BasicData is the basic data that is required for a report, common to all reports
+type BasicData struct {
+	TestName    string `json:"test_name"`
+	CommitOrTag string `json:"commit_or_tag"`
+
+	// Test metrics
+	TestStart time.Time `json:"test_start_timestamp"`
+	TestEnd   time.Time `json:"test_end_timestamp"`
+
+	// all, generator settings, including segments
+	GeneratorConfigs map[string]*wasp.Config `json:"generator_configs"`
+}
+
+func (b *BasicData) Validate() error {
+	if b.TestStart.IsZero() {
+		return errors.New("test start time is missing. We cannot query Loki without a time range. Please set it and try again")
+	}
+	if b.TestEnd.IsZero() {
+		return errors.New("test end time is missing. We cannot query Loki without a time range. Please set it and try again")
+	}
+
+	if len(b.GeneratorConfigs) == 0 {
+		return errors.New("generator configs are missing. At least one is required. Please set them and try again")
+	}
+
+	return nil
+}
+
+func (b *BasicData) IsComparable(otherData BasicData) error {
+	// are all configs present? do they have the same schedule type? do they have the same segments? is call timeout the same? is rate limit timeout the same?
+	if len(b.GeneratorConfigs) != len(otherData.GeneratorConfigs) {
+		return fmt.Errorf("generator configs count is different. Expected %d, got %d", len(b.GeneratorConfigs), len(otherData.GeneratorConfigs))
+	}
+
+	for name1, cfg1 := range b.GeneratorConfigs {
+		if cfg2, ok := otherData.GeneratorConfigs[name1]; !ok {
+			return fmt.Errorf("generator config %s is missing from the other report", name1)
+		} else {
+			if err := compareGeneratorConfigs(cfg1, cfg2); err != nil {
+				return err
+			}
+		}
+	}
+
+	for name2 := range otherData.GeneratorConfigs {
+		if _, ok := b.GeneratorConfigs[name2]; !ok {
+			return fmt.Errorf("generator config %s is missing from the current report", name2)
+		}
+	}
+
+	// TODO: would be good to be able to check if Gun and VU are the same, but idk yet how we could do that easily [hash the code?]
+
+	return nil
+}
+
+func compareGeneratorConfigs(cfg1, cfg2 *wasp.Config) error {
+	if cfg1.LoadType != cfg2.LoadType {
+		return fmt.Errorf("load types are different. Expected %s, got %s", cfg1.LoadType, cfg2.LoadType)
+	}
+
+	if len(cfg1.Schedule) != len(cfg2.Schedule) {
+		return fmt.Errorf("schedules are different. Expected %d, got %d", len(cfg1.Schedule), len(cfg2.Schedule))
+	}
+
+	for i, segment1 := range cfg1.Schedule {
+		segment2 := cfg2.Schedule[i]
+		if segment1 == nil {
+			return fmt.Errorf("schedule at index %d is nil in the current report", i)
+		}
+		if segment2 == nil {
+			return fmt.Errorf("schedule at index %d is nil in the other report", i)
+		}
+		if *segment1 != *segment2 {
+			return fmt.Errorf("schedules at index %d are different. Expected %s, got %s", i, mustMarshallSegment(segment1), mustMarshallSegment(segment2))
+		}
+	}
+
+	if cfg1.CallTimeout != cfg2.CallTimeout {
+		return fmt.Errorf("call timeouts are different. Expected %s, got %s", cfg1.CallTimeout, cfg2.CallTimeout)
+	}
+
+	if cfg1.RateLimitUnitDuration != cfg2.RateLimitUnitDuration {
+		return fmt.Errorf("rate limit unit durations are different. Expected %s, got %s", cfg1.RateLimitUnitDuration, cfg2.RateLimitUnitDuration)
+	}
+
+	return nil
+}
+
+func mustMarshallSegment(segment *wasp.Segment) string {
+	segmentBytes, err := json.MarshalIndent(segment, "", " ")
+	if err != nil {
+		panic(err)
+	}
+
+	return string(segmentBytes)
+}

wasp/benchspy/loki.go

@@ -0,0 +1,147 @@
+package benchspy
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/chainlink-testing-framework/lib/client"
+	"github.com/smartcontractkit/chainlink-testing-framework/wasp"
+)
+
+func NewLokiQuery(queries map[string]string, lokiConfig *wasp.LokiConfig) *LokiQuery {
+	return &LokiQuery{
+		Kind:         "loki",
+		Queries:      queries,
+		LokiConfig:   lokiConfig,
+		QueryResults: make(map[string][]string),
+	}
+}
+
+type LokiQuery struct {
+	Kind string `json:"kind"`
+	// Test metrics
+	StartTime time.Time `json:"start_time"`
+	EndTime   time.Time `json:"end_time"`
+
+	// Performance queries
+	// a map of name to query template, ex: "average cpu usage": "avg(rate(cpu_usage_seconds_total[5m]))"
+	Queries map[string]string `json:"queries"`
+	// Performance queries results
+	// can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc
+	QueryResults map[string][]string `json:"query_results"`
+	// In case something went wrong
+	Errors []error `json:"errors"`
+
+	LokiConfig *wasp.LokiConfig `json:"-"`
+}
+
+func (l *LokiQuery) Results() map[string][]string {
+	return l.QueryResults
+}
+
+func (l *LokiQuery) IsComparable(otherQueryExecutor QueryExecutor) error {
+	otherType := reflect.TypeOf(otherQueryExecutor)
+
+	if otherType != reflect.TypeOf(l) {
+		return fmt.Errorf("expected type %s, got %s", reflect.TypeOf(l), otherType)
+	}
+
+	return l.compareLokiQueries(otherQueryExecutor.(*LokiQuery).Queries)
+}
+
+func (l *LokiQuery) Validate() error {
+	if len(l.Queries) == 0 {
+		return errors.New("there are no Loki queries, there's nothing to fetch. Please set them and try again")
+	}
+	if l.LokiConfig == nil {
+		return errors.New("loki config is missing. Please set it and try again")
+	}
+
+	return nil
+}
+
+func (l *LokiQuery) Execute() error {
+	splitAuth := strings.Split(l.LokiConfig.BasicAuth, ":")
+	var basicAuth client.LokiBasicAuth
+	if len(splitAuth) == 2 {
+		basicAuth = client.LokiBasicAuth{
+			Login:    splitAuth[0],
+			Password: splitAuth[1],
+		}
+	}
+
+	l.QueryResults = make(map[string][]string)
+
+	// TODO this can also be parallelized, just remember to add a mutex for writing to results map
+	for name, query := range l.Queries {
+		queryParams := client.LokiQueryParams{
+			Query:     query,
+			StartTime: l.StartTime,
+			EndTime:   l.EndTime,
+			Limit:     1000, //TODO make this configurable
+		}
+
+		parsedLokiUrl, err := url.Parse(l.LokiConfig.URL)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse Loki URL %s", l.LokiConfig.URL)
+		}
+
+		lokiUrl := parsedLokiUrl.Scheme + "://" + parsedLokiUrl.Host
+		lokiClient := client.NewLokiClient(lokiUrl, l.LokiConfig.TenantID, basicAuth, queryParams)
+
+		ctx, cancelFn := context.WithTimeout(context.Background(), l.LokiConfig.Timeout)
+		rawLogs, err := lokiClient.QueryLogs(ctx)
+		if err != nil {
+			l.Errors = append(l.Errors, err)
+			cancelFn()
+			continue
+		}
+
+		cancelFn()
+		l.QueryResults[name] = []string{}
+		for _, log := range rawLogs {
+			l.QueryResults[name] = append(l.QueryResults[name], log.Log)
+		}
+	}
+
+	if len(l.Errors) > 0 {
+		return errors.New("there were errors while fetching the results. Please check the errors and try again")
+	}
+
+	return nil
+}
+
+func (l *LokiQuery) compareLokiQueries(other map[string]string) error {
+	this := l.Queries
+	if len(this) != len(other) {
+		return fmt.Errorf("queries count is different. Expected %d, got %d", len(this), len(other))
+	}
+
+	for name1, query1 := range this {
+		if query2, ok := other[name1]; !ok {
+			return fmt.Errorf("query %s is missing from the other report", name1)
+		} else {
+			if query1 != query2 {
+				return fmt.Errorf("query %s is different. Expected %s, got %s", name1, query1, query2)
+			}
+		}
+	}
+
+	for name2 := range other {
+		if _, ok := this[name2]; !ok {
+			return fmt.Errorf("query %s is missing from the current report", name2)
+		}
+	}
+
+	return nil
+}
+
+func (l *LokiQuery) TimeRange(start, end time.Time) {
+	l.StartTime = start
+	l.EndTime = end
+}

wasp/benchspy/report.go

@@ -0,0 +1,113 @@
+package benchspy
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// StandardReport is a report that contains all the necessary data for a performance test
+type StandardReport struct {
+	BasicData
+	LocalReportStorage
+	ResourceReporter
+	QueryExecutors []QueryExecutor `json:"query_executors"`
+}
+
+func (b *StandardReport) Store() (string, error) {
+	return b.LocalReportStorage.Store(b.TestName, b.CommitOrTag, b)
+}
+
+func (b *StandardReport) Load() error {
+	return b.LocalReportStorage.Load(b.TestName, b.CommitOrTag, b)
+}
+
+func (b *StandardReport) Fetch() error {
+	basicErr := b.BasicData.Validate()
+	if basicErr != nil {
+		return basicErr
+	}
+
+	// TODO parallelize it
+	for _, queryExecutor := range b.QueryExecutors {
+		queryExecutor.TimeRange(b.TestStart, b.TestEnd)
+
+		if validateErr := queryExecutor.Validate(); validateErr != nil {
+			return validateErr
+		}
+
+		if execErr := queryExecutor.Execute(); execErr != nil {
+			return execErr
+		}
+	}
+
+	resourceErr := b.FetchResources()
+	if resourceErr != nil {
+		return resourceErr
+	}
+
+	return nil
+}
+
+func (b *StandardReport) IsComparable(otherReport StandardReport) error {
+	basicErr := b.BasicData.IsComparable(otherReport.BasicData)
+	if basicErr != nil {
+		return basicErr
+	}
+
+	if resourceErr := b.CompareResources(&otherReport.ResourceReporter); resourceErr != nil {
+		return resourceErr
+	}
+
+	for i, queryExecutor := range b.QueryExecutors {
+		queryErr := queryExecutor.IsComparable(otherReport.QueryExecutors[i])
+		if queryErr != nil {
+			return queryErr
+		}
+	}
+
+	return nil
+}
+
+func (s *StandardReport) UnmarshalJSON(data []byte) error {
+	// helper struct with QueryExecutors as json.RawMessage
+	type Alias StandardReport
+	var raw struct {
+		Alias
+		QueryExecutors []json.RawMessage `json:"query_executors"`
+	}
+
+	// unmarshal into the helper struct to populate other fields automatically
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	var queryExecutors []QueryExecutor
+
+	// manually decide, which QueryExecutor implementation to use based on the "kind" field
+	for _, rawExecutor := range raw.QueryExecutors {
+		var typeIndicator struct {
+			Kind string `json:"kind"`
+		}
+		if err := json.Unmarshal(rawExecutor, &typeIndicator); err != nil {
+			return err
+		}
+
+		var executor QueryExecutor
+		switch typeIndicator.Kind {
+		case "loki":
+			executor = &LokiQuery{}
+		default:
+			return fmt.Errorf("unknown query executor type: %s", typeIndicator.Kind)
+		}
+
+		if err := json.Unmarshal(rawExecutor, executor); err != nil {
+			return err
+		}
+
+		queryExecutors = append(s.QueryExecutors, executor)
+	}
+
+	*s = StandardReport(raw.Alias)
+	s.QueryExecutors = queryExecutors
+	return nil
+}