diff --git a/changes/22069-gitops-async-software-batch b/changes/22069-gitops-async-software-batch new file mode 100644 index 000000000000..35f0652fe209 --- /dev/null +++ b/changes/22069-gitops-async-software-batch @@ -0,0 +1 @@ +* Modified `POST /api/latest/fleet/software/batch` endpoint to be asynchronous and added a new endpoint `GET /api/latest/fleet/software/batch/{request_uuid}` to retrieve the result of the batch upload. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index caf72413da65..eda0660a731d 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -49,6 +49,7 @@ import ( "github.com/fleetdm/fleet/v4/server/pubsub" "github.com/fleetdm/fleet/v4/server/service" "github.com/fleetdm/fleet/v4/server/service/async" + "github.com/fleetdm/fleet/v4/server/service/redis_key_value" "github.com/fleetdm/fleet/v4/server/service/redis_lock" "github.com/fleetdm/fleet/v4/server/service/redis_policy_set" "github.com/fleetdm/fleet/v4/server/sso" @@ -798,6 +799,7 @@ the way that the Fleet server works. softwareInstallStore, bootstrapPackageStore, distributedLock, + redis_key_value.New(redisPool), ) if err != nil { initFatal(err, "initial Fleet Premium service") diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index e775f4ea5fb4..f39ff1cd5551 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -2320,8 +2320,8 @@ func TestGetTeamsYAMLAndApply(t *testing.T) { declaration.DeclarationUUID = uuid.NewString() return declaration, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { - return nil, nil + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil } actualYaml := runAppForTest(t, []string{"get", "teams", "--yaml"}) diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 64cb9fda19cc..b93496155999 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -182,7 +182,8 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -229,7 +230,10 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) { ds.NewJobFunc = func(ctx context.Context, job *fleet.Job) (*fleet.Job, error) { return &fleet.Job{}, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } @@ -285,7 +289,8 @@ func TestGitOpsBasicTeam(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -373,7 +378,10 @@ func TestGitOpsBasicTeam(t *testing.T) { ds.DeleteMDMAppleDeclarationByNameFunc = func(ctx context.Context, teamID *uint, name string) error { return nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error { @@ -644,6 +652,7 @@ func TestGitOpsFullTeam(t *testing.T) { MDMPusher: mockPusher{}, FleetConfig: &fleetCfg, NoCacheDatastore: true, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -804,8 +813,11 @@ func TestGitOpsFullTeam(t *testing.T) { return nil } var appliedSoftwareInstallers []*fleet.UploadSoftwareInstallerPayload - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { appliedSoftwareInstallers = installers + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.SetTeamVPPAppsFunc = func(ctx context.Context, teamID *uint, adamIDs []fleet.VPPAppTeam) error { @@ -937,7 +949,8 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -1055,7 +1068,10 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) { savedTeam = team return team, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { @@ -1201,7 +1217,8 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) // Mock appConfig @@ -1317,7 +1334,10 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) { savedTeam = team return team, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { @@ -1634,9 +1654,9 @@ func TestGitOpsTeamSofwareInstallers(t *testing.T) { file string wantErr string }{ - {"testdata/gitops/team_software_installer_not_found.yml", "Please make sure that URLs are publicy accessible to the internet."}, + {"testdata/gitops/team_software_installer_not_found.yml", "Please make sure that URLs are reachable from your Fleet server."}, {"testdata/gitops/team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe or .deb."}, - {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 500 MB"}, + {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 500 MiB"}, {"testdata/gitops/team_software_installer_valid.yml", ""}, {"testdata/gitops/team_software_installer_valid_apply.yml", ""}, {"testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, @@ -1668,10 +1688,13 @@ func TestGitOpsTeamSoftwareInstallersQueryEnv(t *testing.T) { t.Setenv("QUERY_VAR", "IT_WORKS") - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { if installers[0].PreInstallQuery != "select IT_WORKS" { - return nil, fmt.Errorf("Missing env var, got %s", installers[0].PreInstallQuery) + return fmt.Errorf("Missing env var, got %s", installers[0].PreInstallQuery) } + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } @@ -1686,9 +1709,9 @@ func TestGitOpsNoTeamSoftwareInstallers(t *testing.T) { noTeamFile string wantErr string }{ - {"testdata/gitops/no_team_software_installer_not_found.yml", "Please make sure that URLs are publicy accessible to the internet."}, + {"testdata/gitops/no_team_software_installer_not_found.yml", "Please make sure that URLs are reachable from your Fleet server."}, {"testdata/gitops/no_team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe or .deb."}, - {"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 500 MB"}, + {"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 500 MiB"}, {"testdata/gitops/no_team_software_installer_valid.yml", ""}, {"testdata/gitops/no_team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, {"testdata/gitops/no_team_software_installer_pre_condition_not_found.yml", "no such file or directory"}, @@ -2050,6 +2073,7 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig, FleetConfig: &fleetCfg, License: license, NoCacheDatastore: true, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -2181,7 +2205,10 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig, declaration.DeclarationUUID = uuid.NewString() return declaration, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } @@ -2890,3 +2917,26 @@ software: }) } } + +type memKeyValueStore struct { + m map[string]string +} + +func newMemKeyValueStore() *memKeyValueStore { + return &memKeyValueStore{ + m: make(map[string]string), + } +} + +func (m *memKeyValueStore) Set(ctx context.Context, key string, value string, expireTime time.Duration) error { + m.m[key] = value + return nil +} + +func (m *memKeyValueStore) Get(ctx context.Context, key string) (*string, error) { + v, ok := m.m[key] + if !ok { + return nil, nil + } + return &v, nil +} diff --git a/ee/server/service/mdm_external_test.go b/ee/server/service/mdm_external_test.go index 1d92d1ce3b43..760d046c9c5b 100644 --- a/ee/server/service/mdm_external_test.go +++ b/ee/server/service/mdm_external_test.go @@ -109,6 +109,7 @@ func setupMockDatastorePremiumService(t testing.TB) (*mock.Store, *eeservice.Ser nil, nil, nil, + nil, ) if err != nil { panic(err) diff --git a/ee/server/service/service.go b/ee/server/service/service.go index 7ef6f8b8a530..fb66f21136ad 100644 --- a/ee/server/service/service.go +++ b/ee/server/service/service.go @@ -30,6 +30,7 @@ type Service struct { softwareInstallStore fleet.SoftwareInstallerStore bootstrapPackageStore fleet.MDMBootstrapPackageStore distributedLock fleet.Lock + keyValueStore fleet.KeyValueStore } func NewService( @@ -46,6 +47,7 @@ func NewService( softwareInstallStore fleet.SoftwareInstallerStore, bootstrapPackageStore fleet.MDMBootstrapPackageStore, distributedLock fleet.Lock, + keyValueStore fleet.KeyValueStore, ) (*Service, error) { authorizer, err := authz.NewAuthorizer() if err != nil { @@ -67,6 +69,7 @@ func NewService( softwareInstallStore: softwareInstallStore, bootstrapPackageStore: bootstrapPackageStore, distributedLock: distributedLock, + keyValueStore: keyValueStore, } // Override methods that can't be easily overriden via diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 5a1d67910665..ac4461a592b2 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -14,6 +14,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/pkg/fleethttp" @@ -24,6 +25,7 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mdm/apple/vpp" "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/go-kit/log" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/google/uuid" @@ -1112,13 +1114,21 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f return meta.Extension, nil } -const maxInstallerSizeBytes int64 = 1024 * 1024 * 500 +const ( + maxInstallerSizeBytes int64 = 1024 * 1024 * 500 + batchSoftwarePrefix = "software_batch_" +) func (svc *Service) BatchSetSoftwareInstallers( ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool, -) ([]fleet.SoftwarePackageResponse, error) { +) (string, error) { if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { - return nil, err + return "", err + } + + vc, ok := viewer.FromContext(ctx) + if !ok { + return "", fleet.ErrNoContext } var teamID *uint @@ -1127,98 +1137,165 @@ func (svc *Service) BatchSetSoftwareInstallers( if err != nil { // If this is a dry run, the team may not have been created yet if dryRun && fleet.IsNotFound(err) { - return nil, nil + return "", nil } - return nil, err + return "", err } teamID = &tm.ID } if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: teamID}, fleet.ActionWrite); err != nil { - return nil, ctxerr.Wrap(ctx, err, "validating authorization") + return "", ctxerr.Wrap(ctx, err, "validating authorization") } + // Verify payloads first, to prevent starting the download+upload process if the data is invalid. for _, payload := range payloads { if len(payload.URL) > fleet.SoftwareInstallerURLMaxLength { - return nil, fleet.NewInvalidArgumentError( + return "", fleet.NewInvalidArgumentError( "software.url", "software URL is too long, must be less than 256 characters", ) } + if _, err := url.ParseRequestURI(payload.URL); err != nil { + return "", fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) is invalid", payload.URL), + ) + } } - vc, ok := viewer.FromContext(ctx) - if !ok { - return nil, fleet.ErrNoContext + // keyExpireTime is the current maximum time supported for retrieving + // the result of a software by batch operation. + const keyExpireTime = 24 * time.Hour + + requestUUID := uuid.NewString() + if err := svc.keyValueStore.Set(ctx, batchSoftwarePrefix+requestUUID, batchSetProcessing, keyExpireTime); err != nil { + return "", ctxerr.Wrapf(ctx, err, "failed to set key as %s", batchSetProcessing) } - g, workerCtx := errgroup.WithContext(ctx) - g.SetLimit(3) - // critical to avoid data race, the slice is pre-allocated and each - // goroutine only writes to its index. - installers := make([]*fleet.UploadSoftwareInstallerPayload, len(payloads)) + svc.logger.Log( + "msg", "software batch start", + "request_uuid", requestUUID, + "team_id", teamID, + "payloads", len(payloads), + ) - for i, p := range payloads { - i, p := i, p + go svc.softwareBatchUpload( + requestUUID, + teamID, + vc.UserID(), + payloads, + dryRun, + ) - g.Go(func() error { - // validate the URL before doing the request - _, err := url.ParseRequestURI(p.URL) - if err != nil { - return fleet.NewInvalidArgumentError( + return requestUUID, nil +} + +const ( + batchSetProcessing = "processing" + batchSetCompleted = "completed" + batchSetFailedPrefix = "failed:" +) + +func (svc *Service) softwareBatchUpload( + requestUUID string, + teamID *uint, + userID uint, + payloads []fleet.SoftwareInstallerPayload, + dryRun bool, +) { + var batchErr error + + // We do not use the request ctx on purpose because this method runs in the background. + ctx := context.Background() + + defer func(start time.Time) { + status := batchSetCompleted + if batchErr != nil { + status = fmt.Sprintf("%s%s", batchSetFailedPrefix, batchErr) + } + logger := log.With(svc.logger, + "request_uuid", requestUUID, + "team_id", teamID, + "payloads", len(payloads), + "status", status, + "took", time.Since(start), + ) + logger.Log("msg", "software batch done") + // Give 10m for the client to read the result (it overrides the previos expiration time). + if err := svc.keyValueStore.Set(ctx, batchSoftwarePrefix+requestUUID, status, 10*time.Minute); err != nil { + logger.Log("msg", "failed to set result", "err", err) + } + }(time.Now()) + + downloadURLFn := func(ctx context.Context, url string) (http.Header, []byte, error) { + client := fleethttp.NewClient() + client.Transport = fleethttp.NewSizeLimitTransport(maxInstallerSizeBytes) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, nil, fmt.Errorf("creating request for URL %q: %w", url, err) + } + + resp, err := client.Do(req) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { + return nil, nil, fleet.NewInvalidArgumentError( "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q) is invalid", p.URL), + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MiB", url, maxInstallerSizeBytes/(1024*1024)), ) } - client := fleethttp.NewClient() - client.Transport = fleethttp.NewSizeLimitTransport(maxInstallerSizeBytes) - req, err := http.NewRequestWithContext(workerCtx, http.MethodGet, p.URL, nil) - if err != nil { - return ctxerr.Wrapf(ctx, err, "creating request for URL %s", p.URL) - } + return nil, nil, fmt.Errorf("performing request for URL %q: %w", url, err) + } + defer resp.Body.Close() - resp, err := client.Do(req) - if err != nil { - var maxBytesErr *http.MaxBytesError - if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { - return fleet.NewInvalidArgumentError( - "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MB", p.URL, maxInstallerSizeBytes/(1024*1024)), - ) - } + if resp.StatusCode == http.StatusNotFound { + return nil, nil, fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) returned \"Not Found\". Please make sure that URLs are reachable from your Fleet server.", url), + ) + } - return ctxerr.Wrapf(ctx, err, "performing request for URL %s", p.URL) - } - defer resp.Body.Close() + // Allow all 2xx and 3xx status codes in this pass. + if resp.StatusCode >= 400 { + return nil, nil, fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) received response status code %d.", url, resp.StatusCode), + ) + } - if resp.StatusCode == http.StatusNotFound { - return fleet.NewInvalidArgumentError( + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + // the max size error can be received either at client.Do or here when + // reading the body if it's caught via a limited body reader. + var maxBytesErr *http.MaxBytesError + if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { + return nil, nil, fleet.NewInvalidArgumentError( "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q) doesn't exist. Please make sure that URLs are publicy accessible to the internet.", p.URL), + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MiB", url, maxInstallerSizeBytes/(1024*1024)), ) } + return nil, nil, fmt.Errorf("reading installer %q contents: %w", url, err) + } - // Allow all 2xx and 3xx status codes in this pass. - if resp.StatusCode > 400 { - return fleet.NewInvalidArgumentError( - "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q) received response status code %d.", p.URL, resp.StatusCode), - ) - } + return resp.Header, bodyBytes, nil + } + + var g errgroup.Group + g.SetLimit(3) + // critical to avoid data race, the slice is pre-allocated and each + // goroutine only writes to its index. + installers := make([]*fleet.UploadSoftwareInstallerPayload, len(payloads)) + + for i, p := range payloads { + i, p := i, p - bodyBytes, err := io.ReadAll(resp.Body) + g.Go(func() error { + headers, bodyBytes, err := downloadURLFn(ctx, p.URL) if err != nil { - // the max size error can be received either at client.Do or here when - // reading the body if it's caught via a limited body reader. - var maxBytesErr *http.MaxBytesError - if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { - return fleet.NewInvalidArgumentError( - "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MB", p.URL, maxInstallerSizeBytes/(1024*1024)), - ) - } - return ctxerr.Wrapf(ctx, err, "reading installer %q contents", p.URL) + return err } installer := &fleet.UploadSoftwareInstallerPayload{ @@ -1229,13 +1306,13 @@ func (svc *Service) BatchSetSoftwareInstallers( UninstallScript: p.UninstallScript, InstallerFile: bytes.NewReader(bodyBytes), SelfService: p.SelfService, - UserID: vc.UserID(), + UserID: userID, URL: p.URL, } // set the filename before adding metadata, as it is used as fallback var filename string - cdh, ok := resp.Header["Content-Disposition"] + cdh, ok := headers["Content-Disposition"] if ok && len(cdh) > 0 { _, params, err := mime.ParseMediaType(cdh[0]) if err == nil { @@ -1273,30 +1350,88 @@ func (svc *Service) BatchSetSoftwareInstallers( } if err := g.Wait(); err != nil { - // NOTE: intentionally not wrapping to avoid polluting user - // errors. - return nil, err + // NOTE: intentionally not wrapping to avoid polluting user errors. + batchErr = err + return } if dryRun { - return nil, nil + return } for _, payload := range installers { if err := svc.storeSoftware(ctx, payload); err != nil { - return nil, ctxerr.Wrap(ctx, err, "storing software installer") + batchErr = fmt.Errorf("storing software installer %q: %w", payload.Filename, err) + return } } - insertedSoftwareInstallers, err := svc.ds.BatchSetSoftwareInstallers(ctx, teamID, installers) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "batch set software installers") + if err := svc.ds.BatchSetSoftwareInstallers(ctx, teamID, installers); err != nil { + batchErr = fmt.Errorf("batch set software installers: %w", err) + return } // Note: per @noahtalerman we don't want activity items for CLI actions // anymore, so that's intentionally skipped. +} + +func (svc *Service) GetBatchSetSoftwareInstallersResult(ctx context.Context, tmName string, requestUUID string, dryRun bool) (string, string, []fleet.SoftwarePackageResponse, error) { + // We've already authorized in the POST /api/latest/fleet/software/batch, + // but adding it here so we don't need to worry about a special case endpoint. + if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { + return "", "", nil, err + } + + result, err := svc.keyValueStore.Get(ctx, batchSoftwarePrefix+requestUUID) + if err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "failed to get result") + } + if result == nil { + return "", "", nil, ctxerr.Wrap(ctx, notFoundError{}, "request_uuid not found") + } + + switch { + case *result == batchSetCompleted: + if dryRun { + return fleet.BatchSetSoftwareInstallersStatusCompleted, "", nil, nil + } // this will fall through to retrieving software packages if not a dry run. + case *result == batchSetProcessing: + return fleet.BatchSetSoftwareInstallersStatusProcessing, "", nil, nil + case strings.HasPrefix(*result, batchSetFailedPrefix): + message := strings.TrimPrefix(*result, batchSetFailedPrefix) + return fleet.BatchSetSoftwareInstallersStatusFailed, message, nil, nil + default: + return "", "", nil, ctxerr.New(ctx, "invalid status") + } + + var ( + teamID uint // GetSoftwareInstallers uses 0 for "No team" + ptrTeamID *uint // Authorize uses *uint for "No team" teamID + ) + if tmName != "" { + team, err := svc.ds.TeamByName(ctx, tmName) + if err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "load team by name") + } + teamID = team.ID + ptrTeamID = &team.ID + } + + // We've already authorized in the POST /api/latest/fleet/software/batch, + // but adding it here so we don't need to worry about a special case endpoint. + // + // We use fleet.ActionWrite because this method is the counterpart of the POST + // /api/latest/fleet/software/batch. + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: ptrTeamID}, fleet.ActionWrite); err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "validating authorization") + } + + softwarePackages, err := svc.ds.GetSoftwareInstallers(ctx, teamID) + if err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "get software installers") + } - return insertedSoftwareInstallers, nil + return fleet.BatchSetSoftwareInstallersStatusCompleted, "", softwarePackages, nil } func (svc *Service) SelfServiceInstallSoftwareTitle(ctx context.Context, host *fleet.Host, softwareTitleID uint) error { diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 5aa7a2f11d51..7d7f0169e3ad 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -768,7 +768,7 @@ func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwa return ctxerr.Wrap(ctx, err, "cleanup unused software installers") } -func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { +func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { const upsertSoftwareTitles = ` INSERT INTO software_titles (name, source, browser) @@ -878,23 +878,12 @@ ON DUPLICATE KEY UPDATE url = VALUES(url) ` - const loadInsertedSoftwareInstallers = ` -SELECT - team_id, - title_id, - url -FROM - software_installers -WHERE global_or_team_id = ? -` - // use a team id of 0 if no-team var globalOrTeamID uint if tmID != nil { globalOrTeamID = *tmID } - var insertedSoftwareInstallers []fleet.SoftwarePackageResponse if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // if no installers are provided, just delete whatever was in // the table @@ -1040,15 +1029,11 @@ WHERE global_or_team_id = ? } } - if err := sqlx.SelectContext(ctx, tx, &insertedSoftwareInstallers, loadInsertedSoftwareInstallers, globalOrTeamID); err != nil { - return ctxerr.Wrap(ctx, err, "load inserted software installers") - } - return nil }); err != nil { - return nil, err + return err } - return insertedSoftwareInstallers, nil + return nil } func (ds *Datastore) HasSelfServiceSoftwareInstallers(ctx context.Context, hostPlatform string, hostTeamID *uint) (bool, error) { @@ -1135,3 +1120,21 @@ func (ds *Datastore) UpdateSoftwareInstallerWithoutPackageIDs(ctx context.Contex } return nil } + +func (ds *Datastore) GetSoftwareInstallers(ctx context.Context, teamID uint) ([]fleet.SoftwarePackageResponse, error) { + const loadInsertedSoftwareInstallers = ` +SELECT + team_id, + title_id, + url +FROM + software_installers +WHERE global_or_team_id = ? +` + var softwarePackages []fleet.SoftwarePackageResponse + // Using ds.writer(ctx) on purpose because this method is to be called after applying software. + if err := sqlx.SelectContext(ctx, ds.writer(ctx), &softwarePackages, loadInsertedSoftwareInstallers, teamID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get software installers") + } + return softwarePackages, nil +} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 862d70063a07..178b85807148 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -630,11 +630,15 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { } // batch set with everything empty - softwareInstallers, err := ds.BatchSetSoftwareInstallers(ctx, &team.ID, nil) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, nil) + require.NoError(t, err) + softwareInstallers, err := ds.GetSoftwareInstallers(ctx, team.ID) require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware(nil) - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware(nil) @@ -642,7 +646,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { // add a single installer ins0 := "installer0" ins0File := bytes.NewReader([]byte("installer0")) - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{{ + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{{ InstallScript: "install", InstallerFile: ins0File, StorageID: ins0, @@ -656,6 +660,8 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { URL: "https://example.com", }}) require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) require.Len(t, softwareInstallers, 1) require.NotNil(t, softwareInstallers[0].TeamID) require.Equal(t, team.ID, *softwareInstallers[0].TeamID) @@ -668,7 +674,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { // add a new installer + ins0 installer ins1 := "installer1" ins1File := bytes.NewReader([]byte("installer1")) - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ { InstallScript: "install", InstallerFile: ins0File, @@ -698,6 +704,8 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }, }) require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) require.Len(t, softwareInstallers, 2) require.NotNil(t, softwareInstallers[0].TitleID) require.NotNil(t, softwareInstallers[0].TeamID) @@ -713,7 +721,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }) // remove ins0 - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ { InstallScript: "install", PostInstallScript: "post-install", @@ -728,6 +736,8 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }, }) require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) require.Len(t, softwareInstallers, 1) require.NotNil(t, softwareInstallers[0].TitleID) require.NotNil(t, softwareInstallers[0].TeamID) @@ -737,7 +747,9 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }) // remove everything - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware([]fleet.SoftwareTitle{}) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 9f9a9de50479..99b2cdb7d27c 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1711,7 +1711,8 @@ type Datastore interface { CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore SoftwareInstallerStore, removeCreatedBefore time.Time) error // BatchSetSoftwareInstallers sets the software installers for the given team or no team. - BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*UploadSoftwareInstallerPayload) ([]SoftwarePackageResponse, error) + BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*UploadSoftwareInstallerPayload) error + GetSoftwareInstallers(ctx context.Context, tmID uint) ([]SoftwarePackageResponse, error) // HasSelfServiceSoftwareInstallers returns true if self-service software installers are available for the team or globally. HasSelfServiceSoftwareInstallers(ctx context.Context, platform string, teamID *uint) (bool, error) diff --git a/server/fleet/service.go b/server/fleet/service.go index 8599e464e1b5..24756ebb6d80 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -643,9 +643,15 @@ type Service interface { // GetSoftwareInstallResults gets the results for a particular software install attempt. GetSoftwareInstallResults(ctx context.Context, installUUID string) (*HostSoftwareInstallerResult, error) - // BatchSetSoftwareInstallers replaces the software installers for a specified team. - // Returns the metadata of inserted software installers. - BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []SoftwareInstallerPayload, dryRun bool) ([]SoftwarePackageResponse, error) + // BatchSetSoftwareInstallers asynchronously replaces the software installers for a specified team. + // Returns a request UUID that can be used to track an ongoing batch request (with GetBatchSetSoftwareInstallersResult). + BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []SoftwareInstallerPayload, dryRun bool) (string, error) + // GetBatchSetSoftwareInstallersResult polls for the status of a batch-apply started by BatchSetSoftwareInstallers. + // Return values: + // - 'status': status of the batch-apply which can be "processing", "completed" or "failed". + // - 'message': which contains error information when the status is "failed". + // - 'packages': Contains the list of the applied software packages (when status is "completed"). This is always empty for a dry run. + GetBatchSetSoftwareInstallersResult(ctx context.Context, tmName string, requestUUID string, dryRun bool) (status string, message string, packages []SoftwarePackageResponse, err error) // SelfServiceInstallSoftwareTitle installs a software title // initiated by the user @@ -1120,3 +1126,17 @@ type Service interface { // CalendarWebhook handles incoming calendar callback requests. CalendarWebhook(ctx context.Context, eventUUID string, channelID string, resourceState string) error } + +type KeyValueStore interface { + Set(ctx context.Context, key string, value string, expireTime time.Duration) error + Get(ctx context.Context, key string) (*string, error) +} + +const ( + // BatchSetSoftwareInstallerStatusProcessing is the value returned for an ongoing BatchSetSoftwareInstallers operation. + BatchSetSoftwareInstallersStatusProcessing = "processing" + // BatchSetSoftwareInstallerStatusCompleted is the value returned for a completed BatchSetSoftwareInstallers operation. + BatchSetSoftwareInstallersStatusCompleted = "completed" + // BatchSetSoftwareInstallerStatusFailed is the value returned for a failed BatchSetSoftwareInstallers operation. + BatchSetSoftwareInstallersStatusFailed = "failed" +) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index bc009c4ea30d..a592559bdf5a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -1070,7 +1070,9 @@ type GetSoftwareInstallResultsFunc func(ctx context.Context, resultsUUID string) type CleanupUnusedSoftwareInstallersFunc func(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore, removeCreatedBefore time.Time) error -type BatchSetSoftwareInstallersFunc func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) +type BatchSetSoftwareInstallersFunc func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error + +type GetSoftwareInstallersFunc func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) type HasSelfServiceSoftwareInstallersFunc func(ctx context.Context, platform string, teamID *uint) (bool, error) @@ -2667,6 +2669,9 @@ type DataStore struct { BatchSetSoftwareInstallersFunc BatchSetSoftwareInstallersFunc BatchSetSoftwareInstallersFuncInvoked bool + GetSoftwareInstallersFunc GetSoftwareInstallersFunc + GetSoftwareInstallersFuncInvoked bool + HasSelfServiceSoftwareInstallersFunc HasSelfServiceSoftwareInstallersFunc HasSelfServiceSoftwareInstallersFuncInvoked bool @@ -6369,13 +6374,20 @@ func (s *DataStore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwar return s.CleanupUnusedSoftwareInstallersFunc(ctx, softwareInstallStore, removeCreatedBefore) } -func (s *DataStore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { +func (s *DataStore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { s.mu.Lock() s.BatchSetSoftwareInstallersFuncInvoked = true s.mu.Unlock() return s.BatchSetSoftwareInstallersFunc(ctx, tmID, installers) } +func (s *DataStore) GetSoftwareInstallers(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { + s.mu.Lock() + s.GetSoftwareInstallersFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallersFunc(ctx, tmID) +} + func (s *DataStore) HasSelfServiceSoftwareInstallers(ctx context.Context, platform string, teamID *uint) (bool, error) { s.mu.Lock() s.HasSelfServiceSoftwareInstallersFuncInvoked = true diff --git a/server/service/client_software.go b/server/service/client_software.go index 413e6dc7e9d8..60a0911093f4 100644 --- a/server/service/client_software.go +++ b/server/service/client_software.go @@ -1,7 +1,10 @@ package service import ( + "errors" + "fmt" "net/url" + "time" "github.com/fleetdm/fleet/v4/server/fleet" ) @@ -29,14 +32,38 @@ func (c *Client) ListSoftwareTitles(query string) ([]fleet.SoftwareTitleListResu } func (c *Client) ApplyNoTeamSoftwareInstallers(softwareInstallers []fleet.SoftwareInstallerPayload, opts fleet.ApplySpecOptions) ([]fleet.SoftwarePackageResponse, error) { - verb, path := "POST", "/api/latest/fleet/software/batch" query, err := url.ParseQuery(opts.RawQuery()) if err != nil { return nil, err } + return c.applySoftwareInstallers(softwareInstallers, query, opts.DryRun) +} + +func (c *Client) applySoftwareInstallers(softwareInstallers []fleet.SoftwareInstallerPayload, query url.Values, dryRun bool) ([]fleet.SoftwarePackageResponse, error) { + path := "/api/latest/fleet/software/batch" var resp batchSetSoftwareInstallersResponse - if err := c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, verb, path, &resp, query.Encode()); err != nil { + if err := c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, "POST", path, &resp, query.Encode()); err != nil { return nil, err } - return resp.Packages, nil + if dryRun && resp.RequestUUID == "" { + return nil, nil + } + + requestUUID := resp.RequestUUID + for { + var resp batchSetSoftwareInstallersResultResponse + if err := c.authenticatedRequestWithQuery(nil, "GET", path+"/"+requestUUID, &resp, query.Encode()); err != nil { + return nil, err + } + switch { + case resp.Status == fleet.BatchSetSoftwareInstallersStatusProcessing: + time.Sleep(5 * time.Second) + case resp.Status == fleet.BatchSetSoftwareInstallersStatusFailed: + return nil, errors.New(resp.Message) + case resp.Status == fleet.BatchSetSoftwareInstallersStatusCompleted: + return resp.Packages, nil + default: + return nil, fmt.Errorf("unknown status: %q", resp.Status) + } + } } diff --git a/server/service/client_teams.go b/server/service/client_teams.go index 5c5180a6b700..5d541e903c9b 100644 --- a/server/service/client_teams.go +++ b/server/service/client_teams.go @@ -94,17 +94,12 @@ func (c *Client) ApplyTeamScripts(tmName string, scripts []fleet.ScriptPayload, } func (c *Client) ApplyTeamSoftwareInstallers(tmName string, softwareInstallers []fleet.SoftwareInstallerPayload, opts fleet.ApplySpecOptions) ([]fleet.SoftwarePackageResponse, error) { - verb, path := "POST", "/api/latest/fleet/software/batch" query, err := url.ParseQuery(opts.RawQuery()) if err != nil { return nil, err } query.Add("team_name", tmName) - var resp batchSetSoftwareInstallersResponse - if err := c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, verb, path, &resp, query.Encode()); err != nil { - return nil, err - } - return resp.Packages, nil + return c.applySoftwareInstallers(softwareInstallers, query, opts.DryRun) } func (c *Client) ApplyTeamAppStoreAppsAssociation(tmName string, vppBatchPayload []fleet.VPPBatchPayload, opts fleet.ApplySpecOptions) error { diff --git a/server/service/handler.go b/server/service/handler.go index 21bdd2f7ed57..7012393952bc 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -381,7 +381,10 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.DELETE("/api/_version_/fleet/software/titles/{title_id:[0-9]+}/available_for_install", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) ue.GET("/api/_version_/fleet/software/install/{install_uuid}/results", getSoftwareInstallResultsEndpoint, getSoftwareInstallResultsRequest{}) + // POST /api/_version_/fleet/software/batch is asynchronous, meaning it will start the process of software download+upload in the background + // and will return a request UUID to be used in GET /api/_version_/fleet/software/batch/{request_uuid} to query for the status of the operation. ue.POST("/api/_version_/fleet/software/batch", batchSetSoftwareInstallersEndpoint, batchSetSoftwareInstallersRequest{}) + ue.GET("/api/_version_/fleet/software/batch/{request_uuid}", batchSetSoftwareInstallersResultEndpoint, batchSetSoftwareInstallersResultRequest{}) // App store software ue.GET("/api/_version_/fleet/software/app_store_apps", getAppStoreAppsEndpoint, getAppStoreAppsRequest{}) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 9255e95beff0..cb2a97966846 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -10903,6 +10903,10 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // create an HTTP server to host the software installer handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ruby.deb" { + w.WriteHeader(http.StatusNotFound) + return + } file, err := os.Open(filepath.Join("testdata", "software-installers", "ruby.deb")) require.NoError(t, err) defer file.Close() @@ -10914,11 +10918,28 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { srv := httptest.NewServer(handler) t.Cleanup(srv.Close) + // do a request with a URL that returns a 404. + softwareToInstall = []fleet.SoftwareInstallerPayload{ + {URL: srv.URL + "/not_found.pkg"}, + } + var batchResponse batchSetSoftwareInstallersResponse + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + message := waitBatchSetSoftwareInstallersFailed(t, s, tm.Name, batchResponse.RequestUUID) + require.NotEmpty(t, message) + require.Contains(t, message, fmt.Sprintf("validation failed: software.url Couldn't edit software. URL (\"%s/not_found.pkg\") returned \"Not Found\". Please make sure that URLs are reachable from your Fleet server.", srv.URL)) + // do a request with a valid URL + rubyURL := srv.URL + "/ruby.deb" softwareToInstall = []fleet.SoftwareInstallerPayload{ - {URL: srv.URL}, + {URL: rubyURL}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages := waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) // TODO(roberto): test with a variety of response codes @@ -10929,7 +10950,7 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { require.Len(t, titlesResp.SoftwareTitles, 1) // Check that the URL is set to software installers uploaded via batch. require.NotNil(t, titlesResp.SoftwareTitles[0].SoftwarePackage.PackageURL) - require.Equal(t, srv.URL, *titlesResp.SoftwareTitles[0].SoftwarePackage.PackageURL) + require.Equal(t, rubyURL, *titlesResp.SoftwareTitles[0].SoftwarePackage.PackageURL) // check that platform is set when the installer is created mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { @@ -10942,14 +10963,26 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { }) // same payload doesn't modify anything - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) newTitlesResp := listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, titlesResp, newTitlesResp) // setting self-service to true updates the software title metadata softwareToInstall[0].SelfService = true - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) newTitlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) titlesResp.SoftwareTitles[0].SoftwarePackage.SelfService = ptr.Bool(true) @@ -10957,7 +10990,9 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // empty payload cleans the software items softwareToInstall = []fleet.SoftwareInstallerPayload{} - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Empty(t, packages) titlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, 0, titlesResp.Count) @@ -10967,9 +11002,14 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // Do a request with a valid URL with no team ////////////////////////// softwareToInstall = []fleet.SoftwareInstallerPayload{ - {URL: srv.URL}, + {URL: rubyURL}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.Nil(t, packages[0].TeamID) // check the application status on team 0 titlesResp = listSoftwareTitlesResponse{} @@ -10978,14 +11018,24 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { require.Len(t, titlesResp.SoftwareTitles, 1) // same payload doesn't modify anything - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.Nil(t, packages[0].TeamID) newTitlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(0))) require.Equal(t, titlesResp, newTitlesResp) // setting self-service to true updates the software title metadata softwareToInstall[0].SelfService = true - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.Nil(t, packages[0].TeamID) newTitlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(0))) titlesResp.SoftwareTitles[0].SoftwarePackage.SelfService = ptr.Bool(true) @@ -10993,13 +11043,50 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // empty payload cleans the software items softwareToInstall = []fleet.SoftwareInstallerPayload{} - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Empty(t, packages) titlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(0))) require.Equal(t, 0, titlesResp.Count) require.Len(t, titlesResp.SoftwareTitles, 0) } +func waitBatchSetSoftwareInstallersCompleted(t *testing.T, s *integrationEnterpriseTestSuite, teamName string, requestUUID string) []fleet.SoftwarePackageResponse { + timeout := time.After(1 * time.Minute) + for { + var batchResultResponse batchSetSoftwareInstallersResultResponse + s.DoJSON("GET", "/api/latest/fleet/software/batch/"+requestUUID, nil, http.StatusOK, &batchResultResponse, "team_name", teamName) + if batchResultResponse.Status == fleet.BatchSetSoftwareInstallersStatusCompleted { + return batchResultResponse.Packages + } + select { + case <-timeout: + t.Fatalf("timeout: %s, %s", teamName, requestUUID) + case <-time.After(500 * time.Millisecond): + // OK, continue + } + } +} + +func waitBatchSetSoftwareInstallersFailed(t *testing.T, s *integrationEnterpriseTestSuite, teamName string, requestUUID string) string { + timeout := time.After(1 * time.Minute) + for { + var batchResultResponse batchSetSoftwareInstallersResultResponse + s.DoJSON("GET", "/api/latest/fleet/software/batch/"+requestUUID, nil, http.StatusOK, &batchResultResponse, "team_name", teamName) + if batchResultResponse.Status == fleet.BatchSetSoftwareInstallersStatusFailed { + require.Empty(t, batchResultResponse.Packages) + return batchResultResponse.Message + } + select { + case <-timeout: + t.Fatalf("timeout: %s, %s", teamName, requestUUID) + case <-time.After(500 * time.Millisecond): + // OK, continue + } + } +} + func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffects() { t := s.T() @@ -11030,7 +11117,14 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec softwareToInstall := []fleet.SoftwareInstallerPayload{ {URL: srv.URL}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + var batchResponse batchSetSoftwareInstallersResponse + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages := waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) titlesResp := listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) titleResponse := getSoftwareTitleResponse{} @@ -11068,7 +11162,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec // Switch self-service flag softwareToInstall[0].SelfService = true - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) newTitlesResp := listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, true, *newTitlesResp.SoftwareTitles[0].SoftwarePackage.SelfService) @@ -11082,7 +11182,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec withUpdatedPreinstallQuery := []fleet.SoftwareInstallerPayload{ {URL: srv.URL, PreInstallQuery: "SELECT * FROM os_version"}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedPreinstallQuery}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedPreinstallQuery}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) titleResponse = getSoftwareTitleResponse{} s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, "SELECT * FROM os_version", titleResponse.SoftwareTitle.SoftwarePackage.PreInstallQuery) @@ -11119,7 +11225,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec withUpdatedInstallScript := []fleet.SoftwareInstallerPayload{ {URL: srv.URL, InstallScript: "apt install ruby"}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) // ensure install count is the same, and uploaded_at hasn't changed s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) @@ -11134,7 +11246,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec trailer = " " // add a character to the response for the installer HTTP call to ensure the file hashes differently // update package - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) // ensure install count is zeroed and uploaded_at HAS changed s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) @@ -11198,7 +11316,15 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPolic URL: srv.URL + "/ruby.deb", }, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", team1.Name) + var batchResponse batchSetSoftwareInstallersResponse + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", team1.Name) + packages := waitBatchSetSoftwareInstallersCompleted(t, s, team1.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, team1.ID, *packages[0].TeamID) + require.Equal(t, srv.URL+"/ruby.deb", packages[0].URL) + // team2 has dummy_installer.pkg and ruby.deb. softwareToInstall = []fleet.SoftwareInstallerPayload{ { @@ -11208,7 +11334,20 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPolic URL: srv.URL + "/ruby.deb", }, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", team2.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", team2.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, team2.Name, batchResponse.RequestUUID) + sort.Slice(packages, func(i, j int) bool { + return packages[i].URL < packages[j].URL + }) + require.Len(t, packages, 2) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, team2.ID, *packages[0].TeamID) + require.Equal(t, srv.URL+"/dummy_installer.pkg", packages[0].URL) + require.NotNil(t, packages[1].TitleID) + require.NotNil(t, packages[1].TeamID) + require.Equal(t, team2.ID, *packages[1].TeamID) + require.Equal(t, srv.URL+"/ruby.deb", packages[1].URL) // Associate ruby.deb to policy1Team1. resp := listSoftwareTitlesResponse{} @@ -11238,7 +11377,9 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPolic // Get rid of all installers in team1. softwareToInstall = []fleet.SoftwareInstallerPayload{} - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", team1.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", team1.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, team1.Name, batchResponse.RequestUUID) + require.Len(t, packages, 0) // policy1Team1 should not be associated to any installer. policy1Team1, err = s.ds.Policy(ctx, policy1Team1.ID) diff --git a/server/service/redis_key_value/redis_key_value.go b/server/service/redis_key_value/redis_key_value.go new file mode 100644 index 000000000000..010c24c19cc0 --- /dev/null +++ b/server/service/redis_key_value/redis_key_value.go @@ -0,0 +1,58 @@ +// Package redis_key_value implements a most basic SET & GET key/value store +// where both the key and the value are strings. +package redis_key_value + +import ( + "context" + "errors" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/datastore/redis" + "github.com/fleetdm/fleet/v4/server/fleet" + redigo "github.com/gomodule/redigo/redis" +) + +// RedisKeyValue is a basic key/value store with SET and GET operations +// Items are removed via expiration (defined in the SET operation). +type RedisKeyValue struct { + pool fleet.RedisPool + testPrefix string // for tests, the key prefix to use to avoid conflicts +} + +// New creates a new RedisKeyValue store. +func New(pool fleet.RedisPool) *RedisKeyValue { + return &RedisKeyValue{pool: pool} +} + +// prefix is used to not collide with other key domains (like live queries or calendar locks). +const prefix = "key_value_" + +// Set creates or overrides the given key with the given value. +// Argument expireTime is used to set the expiration of the item +// (when updating, the expiration of the item is updated). +func (r *RedisKeyValue) Set(ctx context.Context, key string, value string, expireTime time.Duration) error { + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) + defer conn.Close() + + if _, err := redigo.String(conn.Do("SET", r.testPrefix+prefix+key, value, "PX", expireTime.Milliseconds())); err != nil { + return ctxerr.Wrap(ctx, err, "redis failed to set") + } + return nil +} + +// Get returns the value for a given key. +// It returns (nil, nil) if the key doesn't exist. +func (r *RedisKeyValue) Get(ctx context.Context, key string) (*string, error) { + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) + defer conn.Close() + + res, err := redigo.String(conn.Do("GET", r.testPrefix+prefix+key)) + if errors.Is(err, redigo.ErrNil) { + return nil, nil + } + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "redis failed to get") + } + return &res, nil +} diff --git a/server/service/redis_key_value/redis_key_value_test.go b/server/service/redis_key_value/redis_key_value_test.go new file mode 100644 index 000000000000..5f410e4a4918 --- /dev/null +++ b/server/service/redis_key_value/redis_key_value_test.go @@ -0,0 +1,92 @@ +package redis_key_value + +import ( + "context" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/datastore/redis/redistest" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/stretchr/testify/require" +) + +func TestRedisKeyValue(t *testing.T) { + for _, f := range []func(*testing.T, *RedisKeyValue){ + testSetGet, + } { + t.Run(test.FunctionName(f), func(t *testing.T) { + t.Run("standalone", func(t *testing.T) { + kv := setupRedis(t, false, false) + f(t, kv) + }) + t.Run("cluster", func(t *testing.T) { + kv := setupRedis(t, true, true) + f(t, kv) + }) + }) + } +} + +func setupRedis(t testing.TB, cluster, redir bool) *RedisKeyValue { + pool := redistest.SetupRedis(t, t.Name(), cluster, redir, true) + return newRedisKeyValueForTest(t, pool) +} + +type testName interface { + Name() string +} + +func newRedisKeyValueForTest(t testName, pool fleet.RedisPool) *RedisKeyValue { + return &RedisKeyValue{ + pool: pool, + testPrefix: t.Name() + ":", + } +} + +func testSetGet(t *testing.T, kv *RedisKeyValue) { + ctx := context.Background() + + result, err := kv.Get(ctx, "foo") + require.NoError(t, err) + require.Nil(t, result) + + err = kv.Set(ctx, "foo", "bar", 5*time.Second) + require.NoError(t, err) + + result, err = kv.Get(ctx, "foo") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "bar", *result) + + err = kv.Set(ctx, "foo", "zoo", 5*time.Second) + require.NoError(t, err) + + result, err = kv.Get(ctx, "foo") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "zoo", *result) + + err = kv.Set(ctx, "boo", "bar", 2*time.Second) + require.NoError(t, err) + result, err = kv.Get(ctx, "boo") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "bar", *result) + + time.Sleep(3 * time.Second) + result, err = kv.Get(ctx, "boo") + require.NoError(t, err) + require.Nil(t, result) + + // Updating an item, updates the expiration time. + err = kv.Set(ctx, "test", "foo", 2*time.Second) + require.NoError(t, err) + err = kv.Set(ctx, "test", "foo", 10*time.Second) + require.NoError(t, err) + time.Sleep(5 * time.Second) + result, err = kv.Get(ctx, "test") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "foo", *result) +} diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 0542d769c805..b10b6a6f4c7f 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -546,27 +546,64 @@ type batchSetSoftwareInstallersRequest struct { } type batchSetSoftwareInstallersResponse struct { - Packages []fleet.SoftwarePackageResponse `json:"packages"` - Err error `json:"error,omitempty"` + RequestUUID string `json:"request_uuid"` + Err error `json:"error,omitempty"` } func (r batchSetSoftwareInstallersResponse) error() error { return r.Err } func batchSetSoftwareInstallersEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*batchSetSoftwareInstallersRequest) - packages, err := svc.BatchSetSoftwareInstallers(ctx, req.TeamName, req.Software, req.DryRun) + requestUUID, err := svc.BatchSetSoftwareInstallers(ctx, req.TeamName, req.Software, req.DryRun) if err != nil { return batchSetSoftwareInstallersResponse{Err: err}, nil } - return batchSetSoftwareInstallersResponse{Packages: packages}, nil + return batchSetSoftwareInstallersResponse{RequestUUID: requestUUID}, nil } -func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool) ([]fleet.SoftwarePackageResponse, error) { +func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool) (string, error) { // skipauth: No authorization check needed due to implementation returning // only license error. svc.authz.SkipAuthorization(ctx) - return nil, fleet.ErrMissingLicense + return "", fleet.ErrMissingLicense +} + +type batchSetSoftwareInstallersResultRequest struct { + RequestUUID string `url:"request_uuid"` + TeamName string `query:"team_name,optional"` + DryRun bool `query:"dry_run,optional"` // if true, apply validation but do not save changes +} + +type batchSetSoftwareInstallersResultResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Packages []fleet.SoftwarePackageResponse `json:"packages"` + + Err error `json:"error,omitempty"` +} + +func (r batchSetSoftwareInstallersResultResponse) error() error { return r.Err } + +func batchSetSoftwareInstallersResultEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*batchSetSoftwareInstallersResultRequest) + status, message, packages, err := svc.GetBatchSetSoftwareInstallersResult(ctx, req.TeamName, req.RequestUUID, req.DryRun) + if err != nil { + return batchSetSoftwareInstallersResultResponse{Err: err}, nil + } + return batchSetSoftwareInstallersResultResponse{ + Status: status, + Message: message, + Packages: packages, + }, nil +} + +func (svc *Service) GetBatchSetSoftwareInstallersResult(ctx context.Context, tmName string, requestUUID string, dryRun bool) (string, string, []fleet.SoftwarePackageResponse, error) { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return "", "", nil, fleet.ErrMissingLicense } ////////////////////////////////////////////////////////////////////////////// diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 674f6c4441a7..7e5937c56c1e 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -34,6 +34,7 @@ import ( "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/service/async" "github.com/fleetdm/fleet/v4/server/service/mock" + "github.com/fleetdm/fleet/v4/server/service/redis_key_value" "github.com/fleetdm/fleet/v4/server/service/redis_lock" "github.com/fleetdm/fleet/v4/server/sso" "github.com/fleetdm/fleet/v4/server/test" @@ -72,6 +73,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf softwareInstallStore fleet.SoftwareInstallerStore bootstrapPackageStore fleet.MDMBootstrapPackageStore distributedLock fleet.Lock + keyValueStore fleet.KeyValueStore ) if len(opts) > 0 { if opts[0].Clock != nil { @@ -79,6 +81,10 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf } } + if len(opts) > 0 && opts[0].KeyValueStore != nil { + keyValueStore = opts[0].KeyValueStore + } + task := async.NewTask(ds, nil, c, config.OsqueryConfig{}) if len(opts) > 0 { if opts[0].Task != nil { @@ -99,6 +105,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf ssoStore = sso.NewSessionStore(opts[0].Pool) profMatcher = apple_mdm.NewProfileMatcher(opts[0].Pool) distributedLock = redis_lock.NewLock(opts[0].Pool) + keyValueStore = redis_key_value.New(opts[0].Pool) } if opts[0].ProfileMatcher != nil { profMatcher = opts[0].ProfileMatcher @@ -203,6 +210,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf softwareInstallStore, bootstrapPackageStore, distributedLock, + keyValueStore, ) if err != nil { panic(err) @@ -317,6 +325,7 @@ type TestServerOpts struct { NoCacheDatastore bool SoftwareInstallStore fleet.SoftwareInstallerStore BootstrapPackageStore fleet.MDMBootstrapPackageStore + KeyValueStore fleet.KeyValueStore } func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServerOpts) (map[string]fleet.User, *httptest.Server) {