Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make software batch endpoint asynchronous #22258

Merged
merged 2 commits into from
Sep 20, 2024
Merged

Make software batch endpoint asynchronous #22258

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7106f35
Make software batch endpoint to be asynchronous
lucasmrod Sep 19, 2024
9e5333b
Address PR comments
lucasmrod Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/22069-gitops-async-software-batch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Modified `POST /api/latest/fleet/software/batch` endpoint to be asynchronous and added a new endpoint `GET /api/latest/fleet/software/batch/{request_uuid}` to retrieve the result of the batch upload.
2 changes: 2 additions & 0 deletions cmd/fleet/serve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import (
"github.com/fleetdm/fleet/v4/server/pubsub"
"github.com/fleetdm/fleet/v4/server/service"
"github.com/fleetdm/fleet/v4/server/service/async"
"github.com/fleetdm/fleet/v4/server/service/redis_key_value"
"github.com/fleetdm/fleet/v4/server/service/redis_lock"
"github.com/fleetdm/fleet/v4/server/service/redis_policy_set"
"github.com/fleetdm/fleet/v4/server/sso"
Expand Down Expand Up @@ -798,6 +799,7 @@ the way that the Fleet server works.
softwareInstallStore,
bootstrapPackageStore,
distributedLock,
redis_key_value.New(redisPool),
)
if err != nil {
initFatal(err, "initial Fleet Premium service")
Expand Down
4 changes: 2 additions & 2 deletions cmd/fleetctl/get_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2320,8 +2320,8 @@ func TestGetTeamsYAMLAndApply(t *testing.T) {
declaration.DeclarationUUID = uuid.NewString()
return declaration, nil
}
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
return nil
}

actualYaml := runAppForTest(t, []string{"get", "teams", "--yaml"})
Expand Down
74 changes: 62 additions & 12 deletions cmd/fleetctl/gitops_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) {
license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)}
_, ds := runServerWithMockedDS(
t, &service.TestServerOpts{
License: license,
License: license,
KeyValueStore: newMemKeyValueStore(),
},
)

Expand Down Expand Up @@ -229,7 +230,10 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) {
ds.NewJobFunc = func(ctx context.Context, job *fleet.Job) (*fleet.Job, error) {
return &fleet.Job{}, nil
}
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}

Expand Down Expand Up @@ -285,7 +289,8 @@ func TestGitOpsBasicTeam(t *testing.T) {
license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)}
_, ds := runServerWithMockedDS(
t, &service.TestServerOpts{
License: license,
License: license,
KeyValueStore: newMemKeyValueStore(),
},
)

Expand Down Expand Up @@ -373,7 +378,10 @@ func TestGitOpsBasicTeam(t *testing.T) {
ds.DeleteMDMAppleDeclarationByNameFunc = func(ctx context.Context, teamID *uint, name string) error {
return nil
}
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}
ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error {
Expand Down Expand Up @@ -644,6 +652,7 @@ func TestGitOpsFullTeam(t *testing.T) {
MDMPusher: mockPusher{},
FleetConfig: &fleetCfg,
NoCacheDatastore: true,
KeyValueStore: newMemKeyValueStore(),
},
)

Expand Down Expand Up @@ -804,8 +813,11 @@ func TestGitOpsFullTeam(t *testing.T) {
return nil
}
var appliedSoftwareInstallers []*fleet.UploadSoftwareInstallerPayload
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
appliedSoftwareInstallers = installers
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}
ds.SetTeamVPPAppsFunc = func(ctx context.Context, teamID *uint, adamIDs []fleet.VPPAppTeam) error {
Expand Down Expand Up @@ -937,7 +949,8 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) {
license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)}
_, ds := runServerWithMockedDS(
t, &service.TestServerOpts{
License: license,
License: license,
KeyValueStore: newMemKeyValueStore(),
},
)

Expand Down Expand Up @@ -1055,7 +1068,10 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) {
savedTeam = team
return team, nil
}
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}
ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) {
Expand Down Expand Up @@ -1201,7 +1217,8 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) {
license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)}
_, ds := runServerWithMockedDS(
t, &service.TestServerOpts{
License: license,
License: license,
KeyValueStore: newMemKeyValueStore(),
},
)
// Mock appConfig
Expand Down Expand Up @@ -1317,7 +1334,10 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) {
savedTeam = team
return team, nil
}
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}
ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) {
Expand Down Expand Up @@ -1668,10 +1688,13 @@ func TestGitOpsTeamSoftwareInstallersQueryEnv(t *testing.T) {

t.Setenv("QUERY_VAR", "IT_WORKS")

ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
if installers[0].PreInstallQuery != "select IT_WORKS" {
return nil, fmt.Errorf("Missing env var, got %s", installers[0].PreInstallQuery)
return fmt.Errorf("Missing env var, got %s", installers[0].PreInstallQuery)
}
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}

Expand Down Expand Up @@ -2050,6 +2073,7 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig,
FleetConfig: &fleetCfg,
License: license,
NoCacheDatastore: true,
KeyValueStore: newMemKeyValueStore(),
},
)

Expand Down Expand Up @@ -2181,7 +2205,10 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig,
declaration.DeclarationUUID = uuid.NewString()
return declaration, nil
}
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) {
ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
return nil
}
ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) {
return nil, nil
}

Expand Down Expand Up @@ -2890,3 +2917,26 @@ software:
})
}
}

type memKeyValueStore struct {
m map[string]string
}

func newMemKeyValueStore() *memKeyValueStore {
return &memKeyValueStore{
m: make(map[string]string),
}
}

func (m *memKeyValueStore) Set(ctx context.Context, key string, value string, expireTime time.Duration) error {
m.m[key] = value
return nil
}

func (m *memKeyValueStore) Get(ctx context.Context, key string) (*string, error) {
v, ok := m.m[key]
if !ok {
return nil, nil
}
return &v, nil
}
1 change: 1 addition & 0 deletions ee/server/service/mdm_external_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ func setupMockDatastorePremiumService(t testing.TB) (*mock.Store, *eeservice.Ser
nil,
nil,
nil,
nil,
)
if err != nil {
panic(err)
Expand Down
3 changes: 3 additions & 0 deletions ee/server/service/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type Service struct {
softwareInstallStore fleet.SoftwareInstallerStore
bootstrapPackageStore fleet.MDMBootstrapPackageStore
distributedLock fleet.Lock
keyValueStore fleet.KeyValueStore
}

func NewService(
Expand All @@ -46,6 +47,7 @@ func NewService(
softwareInstallStore fleet.SoftwareInstallerStore,
bootstrapPackageStore fleet.MDMBootstrapPackageStore,
distributedLock fleet.Lock,
keyValueStore fleet.KeyValueStore,
) (*Service, error) {
authorizer, err := authz.NewAuthorizer()
if err != nil {
Expand All @@ -67,6 +69,7 @@ func NewService(
softwareInstallStore: softwareInstallStore,
bootstrapPackageStore: bootstrapPackageStore,
distributedLock: distributedLock,
keyValueStore: keyValueStore,
}

// Override methods that can't be easily overriden via
Expand Down
Loading
Loading