diff --git a/go.mod b/go.mod index 125a6950e..90013661a 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c github.com/tmc/dot v0.0.0-20210901225022-f9bc17da75c0 + github.com/xanzy/go-gitlab v0.105.0 go.lsp.dev/uri v0.3.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 diff --git a/go.sum b/go.sum index 553c7c444..400b70d8c 100644 --- a/go.sum +++ b/go.sum @@ -1150,6 +1150,8 @@ github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 h1:jIVmlAFIq github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651/go.mod h1:b26F2tHLqaoRQf8DywqzVaV1MQ9yvjb0OMcNl7Nxu20= github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0 h1:0KGbf+0SMg+UFy4e1A/CPVvXn21f1qtWdeJwxZFoQG8= github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= +github.com/xanzy/go-gitlab v0.105.0 h1:3nyLq0ESez0crcaM19o5S//SvezOQguuIHZ3wgX64hM= +github.com/xanzy/go-gitlab v0.105.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= diff --git a/pkg/checks/update.go b/pkg/checks/update.go index 30343d4fa..806617d9b 100644 --- a/pkg/checks/update.go +++ b/pkg/checks/update.go @@ -38,6 +38,7 @@ func SetupUpdate(ctx context.Context) (*update.Options, lint.EvalRuleErrors) { o := update.New(ctx) o.GithubReleaseQuery = true o.ReleaseMonitoringQuery = true + o.GitlabReleaseQuery = true o.ErrorMessages = make(map[string]string) o.Logger = log.New(log.Writer(), "wolfictl check update: ", log.LstdFlags|log.Lmsgprefix) checkErrors := make(lint.EvalRuleErrors, 0) @@ -121,8 +122,8 @@ func validateUpdateConfig(ctx context.Context, files []string, checkErrors *lint // ensure a backend has been configured if c.Update.Enabled { - if c.Update.ReleaseMonitor == nil && c.Update.GitHubMonitor == nil { - addCheckError(checkErrors, fmt.Errorf("config %s has update config enabled but no release-monitor or github backend monitor configured, see examples in this repository", file)) + if c.Update.ReleaseMonitor == nil && c.Update.GitHubMonitor == nil && c.Update.GitLabMonitor == nil { + addCheckError(checkErrors, fmt.Errorf("config %s has update config enabled but no release-monitor , github backend monitor or gitlab api monitor configured, see examples in this repository", file)) continue } } diff --git a/pkg/cli/update.go b/pkg/cli/update.go index 8f6463b8f..fb10ff295 100644 --- a/pkg/cli/update.go +++ b/pkg/cli/update.go @@ -15,6 +15,7 @@ type options struct { pullRequestTitle string dryRun bool githubReleaseQuery bool + gitlabReleaseQuery bool releaseMonitoringQuery bool useGitSign bool createIssues bool @@ -38,6 +39,7 @@ func cmdUpdate() *cobra.Command { cmd.Flags().BoolVar(&o.dryRun, "dry-run", false, "prints proposed package updates rather than creating a pull request") cmd.Flags().BoolVar(&o.githubReleaseQuery, "github-release-query", true, "query the GitHub graphql API for latest releases") + cmd.Flags().BoolVar(&o.gitlabReleaseQuery, "gitlab-release-query", true, "query the GitLab REST API for latest releases version") cmd.Flags().BoolVar(&o.releaseMonitoringQuery, "release-monitoring-query", true, "query https://release-monitoring.org/ API for latest releases") cmd.Flags().StringArrayVar(&o.packageNames, "package-name", []string{}, "Optional: provide a specific package name to check for updates rather than searching all packages in a repo URI") cmd.Flags().StringVar(&o.pullRequestBaseBranch, "pull-request-base-branch", "main", "base branch to create a pull request against") @@ -74,6 +76,7 @@ func (o options) UpdateCmd(ctx context.Context, repoURI string) error { updateContext.PullRequestTitle = o.pullRequestTitle updateContext.ReleaseMonitoringQuery = o.releaseMonitoringQuery updateContext.GithubReleaseQuery = o.githubReleaseQuery + updateContext.GitlabReleaseQuery = o.gitlabReleaseQuery updateContext.UseGitSign = o.useGitSign updateContext.CreateIssues = o.createIssues updateContext.IssueLabels = o.issueLabels diff --git a/pkg/update/githubReleases.go b/pkg/update/githubReleases.go index 0f1866f31..1ffc62cdd 100644 --- a/pkg/update/githubReleases.go +++ b/pkg/update/githubReleases.go @@ -555,7 +555,7 @@ func (o GitHubReleaseOptions) getLatestVersion(packageNameHash string, versionRe return nil } -func (o GitHubReleaseOptions) shouldSkipVersion(v string) bool { +func shouldSkipVersion(v string) bool { invalid := []string{"alpha", "beta", "rc", "pre"} for _, i := range invalid { if strings.Contains(strings.ToLower(v), i) { @@ -639,7 +639,7 @@ func (o GitHubReleaseOptions) prepareVersion(nameHash, v, id string) (string, er v = strings.ReplaceAll(v, c.Update.VersionSeparator, ".") } - if o.shouldSkipVersion(v) { + if shouldSkipVersion(v) { return "", nil } diff --git a/pkg/update/githubReleases_test.go b/pkg/update/githubReleases_test.go index e3215bcce..526387943 100644 --- a/pkg/update/githubReleases_test.go +++ b/pkg/update/githubReleases_test.go @@ -3,7 +3,6 @@ package update import ( "encoding/json" "fmt" - "log" "os" "path/filepath" "strings" @@ -226,11 +225,7 @@ func TestGitHubReleaseOptions_isVersionPreRelease(t *testing.T) { } for _, tt := range tests { t.Run(tt.version, func(t *testing.T) { - o := GitHubReleaseOptions{ - Logger: log.New(log.Writer(), "test: ", log.LstdFlags|log.Lmsgprefix), - } - - assert.Equalf(t, tt.skip, o.shouldSkipVersion(tt.version), "isVersionPreRelease(%v)", tt.version) + assert.Equalf(t, tt.skip, shouldSkipVersion(tt.version), "isVersionPreRelease(%v)", tt.version) }) } } diff --git a/pkg/update/gitlabReleases.go b/pkg/update/gitlabReleases.go new file mode 100644 index 000000000..04041c7d6 --- /dev/null +++ b/pkg/update/gitlabReleases.go @@ -0,0 +1,223 @@ +package update + +import ( + "errors" + "fmt" + "log" + "os" + "strings" + + "chainguard.dev/melange/pkg/config" + "github.com/Masterminds/semver/v3" + "github.com/wolfi-dev/wolfictl/pkg/melange" + "github.com/xanzy/go-gitlab" +) + +const ( + gitlabBaseURL = "https://gitlab.com" +) + +type GitLabReleaseOptions struct { + PackageConfigs map[string]*melange.Packages + gitlabClient *gitlab.Client + Logger *log.Logger + + ErrorMessages map[string]string +} +type VersionComit struct { + Version string + Commit string +} + +func NewGitlabReleaseOptions(packageConfigs map[string]*melange.Packages) GitLabReleaseOptions { + + token := os.Getenv("GITLAB_TOKEN") + if token == "" { + log.Fatalf("GITLAB_TOKEN environment variable not set") + } + + client, err := gitlab.NewClient(token, gitlab.WithBaseURL(gitlabBaseURL)) + if err != nil { + log.Fatalf("Failed to create gitlab client: %v", err) + } + + o := GitLabReleaseOptions{ + PackageConfigs: packageConfigs, + gitlabClient: client, + Logger: log.New(log.Writer(), "wolfictl check update: ", log.LstdFlags|log.Lmsgprefix), + ErrorMessages: make(map[string]string), + } + + return o +} + +func (o GitLabReleaseOptions) getLatestGitLabVersions() (map[string]NewVersionResults, map[string]string, error) { + if len(o.PackageConfigs) == 0 { + return nil, o.ErrorMessages, errors.New("No package configs provided") + } + + releaseRepoList, tagRepoList := o.getSeparateRepoLists() + + latestVersionResults := make(map[string]NewVersionResults) + + if len(releaseRepoList) > 0 { + o.Logger.Println("Checking for latest new releases") + for packageName, identifier := range releaseRepoList { + o.Logger.Printf("Checking for latest release on %s using identifier %s\n", packageName, identifier) + listReleaseOption := &gitlab.ListReleasesOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 20, + Page: 1, + }, + } + releases, resp, err := o.gitlabClient.Releases.ListReleases(identifier, listReleaseOption) + if err != nil || resp.StatusCode != 200 { + o.ErrorMessages[packageName] = fmt.Sprintf("failed to list releases for %s: %v", packageName, err) + continue + } + if len(releases) == 0 { + o.ErrorMessages[packageName] = fmt.Sprintf("No releases found for %s", packageName) + continue + } + + // filter out releases that match the ignore regex patterns and other filters + allReleaseList := []VersionComit{} + for _, release := range releases { + allReleaseList = append(allReleaseList, VersionComit{ + Version: release.TagName, + Commit: release.Commit.ID, + }) + } + v, c, err := prepareLatestVersion(allReleaseList, &o.PackageConfigs[packageName].Config) + if err != nil { + o.ErrorMessages[packageName] = fmt.Sprintf("Failed to prepare version for %s: %v", packageName, err) + continue + } + latestVersionResults[packageName] = NewVersionResults{ + Version: v, + Commit: c, + } + } + } + + if len(tagRepoList) > 0 { + o.Logger.Println("Checking for latest new tags") + listTagsOption := &gitlab.ListTagsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 50, + Page: 1, + OrderBy: "version", + }, + } + for packageName, identifier := range tagRepoList { + o.Logger.Printf("Checking for latest tag on %s using projectID %s\n", packageName, identifier) + tags, resp, err := o.gitlabClient.Tags.ListTags(identifier, listTagsOption) + if err != nil || resp.StatusCode != 200 { + o.ErrorMessages[packageName] = fmt.Sprintf("Failed to list tags for %s: %v", packageName, err) + continue + } + if len(tags) == 0 { + o.ErrorMessages[packageName] = fmt.Sprintf("No tags found for %s", packageName) + continue + } + + // filter out releases that match the ignore regex patterns and other filters + allTagsList := []VersionComit{} + for _, tag := range tags { + allTagsList = append(allTagsList, VersionComit{ + Version: tag.Name, + Commit: tag.Commit.ID, + }) + } + v, c, err := prepareLatestVersion(allTagsList, &o.PackageConfigs[packageName].Config) + if err != nil { + o.ErrorMessages[packageName] = fmt.Sprintf("Failed to prepare version for %s: %v", packageName, err) + continue + } + + latestVersionResults[packageName] = NewVersionResults{ + Version: v, + Commit: c, + } + } + } + + return latestVersionResults, o.ErrorMessages, nil +} + +func prepareLatestVersion(versionList []VersionComit, packageConfig *config.Configuration) (latestVersion, commit string, err error) { + if len(versionList) == 0 { + return "", "", errors.New("no versions found, empty list of tags/releases") + } + + glm := packageConfig.Update.GitLabMonitor + if glm == nil { + return "", "", errors.New("no gitlab update configuration found for package") + } + + var highestVersion *semver.Version + for _, vc := range versionList { + // Check if version should be ignored based on regex patterns + ignore, err := ignoreVersions(packageConfig.Update.IgnoreRegexPatterns, vc.Version) + if err != nil { + return "", "", err + } + if ignore { + continue + } + + // filters + if shouldSkipVersion(vc.Version) || (glm.TagFilterPrefix != "" && !strings.HasPrefix(vc.Version, glm.TagFilterPrefix)) || (glm.TagFilterContains != "" && !strings.Contains(vc.Version, glm.TagFilterContains)) { + continue + } + + // Parse the version for comparison + currentVersion, err := semver.NewVersion(vc.Version) + if err != nil { + continue // Skip versions that cannot be parsed + } + + // Compare and find the highest version + if highestVersion == nil || currentVersion.GreaterThan(highestVersion) { + highestVersion = currentVersion + latestVersion = vc.Version + commit = vc.Commit + } + } + + if highestVersion == nil { + return "", "", errors.New("no latest version found") + } + + // Apply transformations + if glm.StripPrefix != "" { + latestVersion = strings.TrimPrefix(latestVersion, glm.StripPrefix) + } + if glm.StripSuffix != "" { + latestVersion = strings.TrimSuffix(latestVersion, glm.StripSuffix) + } + + transformedVersion, err := transformVersion(packageConfig.Update, latestVersion) + if err != nil { + return "", "", fmt.Errorf("failed to apply version transforms to %s. error: %s", latestVersion, err) + } + + return transformedVersion, commit, nil +} + +func (o GitLabReleaseOptions) getSeparateRepoLists() (releaseRepoList, tagRepoList map[string]string) { + tagRepoList = make(map[string]string) + releaseRepoList = make(map[string]string) + for _, pc := range o.PackageConfigs { + if monitor := pc.Config.Update.GitLabMonitor; monitor != nil { + identifire := monitor.Identifier + if monitor.UseTags { + tagRepoList[pc.Config.Package.Name] = identifire + } else { + releaseRepoList[pc.Config.Package.Name] = identifire + } + } + } + + return releaseRepoList, tagRepoList +} diff --git a/pkg/update/gitlabReleases_test.go b/pkg/update/gitlabReleases_test.go new file mode 100644 index 000000000..261d50048 --- /dev/null +++ b/pkg/update/gitlabReleases_test.go @@ -0,0 +1,135 @@ +package update + +import ( + "testing" + + "chainguard.dev/melange/pkg/config" +) + +func TestGitLabMonitor_prepareLatestVersion(t *testing.T) { + tests := []struct { + name string + versionList []VersionComit + packageConfig *config.Configuration + expectedVersion string + expectedCommit string + expectError bool + }{ + { + name: "empty version list", + versionList: []VersionComit{}, + packageConfig: &config.Configuration{}, + expectError: true, + }, + { + name: "version ignored by regex", + versionList: []VersionComit{{Version: "v1.0.0-beta", Commit: "abc123"}}, + packageConfig: &config.Configuration{ + Update: config.Update{ + IgnoreRegexPatterns: []string{"beta"}, + }, + }, + expectError: true, + }, + { + name: "negative version ignored by regex", + versionList: []VersionComit{ + {Version: "v1.0.0-beta", Commit: "abc123"}, + {Version: "v1.0.0", Commit: "def456"}, + }, + packageConfig: &config.Configuration{ + Update: config.Update{ + IgnoreRegexPatterns: []string{"beta"}, + GitLabMonitor: &config.GitLabMonitor{}, + }, + }, + expectedVersion: "v1.0.0", + expectedCommit: "def456", + expectError: false, + }, + { + name: "valid version found", + versionList: []VersionComit{{Version: "v1.0.0", Commit: "def456"}}, + packageConfig: &config.Configuration{ + Update: config.Update{ + GitLabMonitor: &config.GitLabMonitor{}, + }, + }, + expectedVersion: "v1.0.0", + expectedCommit: "def456", + expectError: false, + }, + { + name: "multiple versions with proper latest picked", + versionList: []VersionComit{ + {Version: "2.0.1-alpha", Commit: "ghi789"}, + {Version: "1.0.0", Commit: "def456"}, + {Version: "1.5.0", Commit: "jkl012"}, + {Version: "2.0.0", Commit: "mno345"}, + }, + packageConfig: &config.Configuration{ + Update: config.Update{ + IgnoreRegexPatterns: []string{"alpha"}, // Ignoring alpha versions + GitLabMonitor: &config.GitLabMonitor{}, + }, + }, + expectedVersion: "2.0.0", + expectedCommit: "mno345", + expectError: false, + }, + { + name: "multiple versions with proper latest picked and prefix stripped", + versionList: []VersionComit{ + {Version: "v2.0.0-alpha", Commit: "ghi789"}, + {Version: "v1.0.0", Commit: "def456"}, + {Version: "v1.5.0", Commit: "jkl012"}, + }, + packageConfig: &config.Configuration{ + Update: config.Update{ + IgnoreRegexPatterns: []string{"alpha"}, // Ignoring alpha versions + GitLabMonitor: &config.GitLabMonitor{ + StripPrefix: "v", + }, + }, + }, + expectedVersion: "1.5.0", + expectedCommit: "jkl012", + expectError: false, + }, + { + name: "multiple versions with proper latest picked and suffix stripped", + versionList: []VersionComit{ + {Version: "1.0.0-rs", Commit: "def456"}, + {Version: "2.0.0-rs", Commit: "ghi789"}, + {Version: "1.5.0-rs", Commit: "jkl012"}, + }, + packageConfig: &config.Configuration{ + Update: config.Update{ + IgnoreRegexPatterns: []string{"alpha"}, // Ignoring alpha versions + GitLabMonitor: &config.GitLabMonitor{ + StripSuffix: "-rs", + }, + }, + }, + expectedVersion: "2.0.0", + expectedCommit: "ghi789", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + version, commit, err := prepareLatestVersion(tt.versionList, tt.packageConfig) + if (err != nil) != tt.expectError { + t.Errorf("prepareLatestVersion() error = %v, expectError %v", err, tt.expectError) + return + } + if version != tt.expectedVersion { + t.Errorf("Expected version %v, got %v", tt.expectedVersion, version) + } + if commit != tt.expectedCommit { + t.Errorf("Expected commit %v, got %v", tt.expectedCommit, commit) + } + }) + } +} diff --git a/pkg/update/update.go b/pkg/update/update.go index 52b962a20..200b8ee48 100644 --- a/pkg/update/update.go +++ b/pkg/update/update.go @@ -45,6 +45,7 @@ type Options struct { DryRun bool ReleaseMonitoringQuery bool GithubReleaseQuery bool + GitlabReleaseQuery bool UseGitSign bool CreateIssues bool ReleaseMonitorClient *http2.RLHTTPClient @@ -248,6 +249,18 @@ func (o *Options) GetLatestVersions(ctx context.Context, dir string, packageName maps.Copy(latestVersions, v) } + if o.GithubReleaseQuery { + // get latest versions from gitlab(using the same client for now) + glo := NewGitlabReleaseOptions(o.PackageConfigs) + + v, errorMessages, err := glo.getLatestGitLabVersions() + if err != nil { + return nil, fmt.Errorf("failed gitlab releases: %w", err) + } + maps.Copy(o.ErrorMessages, errorMessages) + maps.Copy(latestVersions, v) + } + if o.ReleaseMonitoringQuery { // get latest versions from https://release-monitoring.org/ m := MonitorService{