Skip to content

Commit

Permalink
Updated software_titles unique index idx_sw_titles to use bundle
Browse files Browse the repository at this point in the history
identifier when present

This allows software with different names but the same bundle identifier
to be grouped under the same title. It also allows for software with the
same name but different bundle identifiers to be under two separate
titles.
  • Loading branch information
ksykulev committed Jan 28, 2025
1 parent 9b70a2c commit b0457a1
Show file tree
Hide file tree
Showing 6 changed files with 250 additions and 5 deletions.
1 change: 1 addition & 0 deletions changes/25235-software-titles-uniqueness
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Updated software_titles unique index `idx_sw_titles` to use `bundle_identifier` when present instead of `name`.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package tables

import (
"database/sql"
"fmt"
)

func init() {
MigrationClient.AddMigration(Up_20250124194347, Down_20250124194347)
}

func Up_20250124194347(tx *sql.Tx) error {
if _, err := tx.Exec(`
ALTER TABLE software_titles
ADD COLUMN coalesce_bundle_name VARCHAR(255) GENERATED ALWAYS AS (COALESCE(bundle_identifier, name)) STORED;
`); err != nil {
return fmt.Errorf("failed to add generated column: %w", err)
}

if _, err := tx.Exec(`
ALTER TABLE software_titles
DROP INDEX idx_sw_titles,
ADD UNIQUE INDEX idx_sw_titles (coalesce_bundle_name, source, browser);
`); err != nil {
return fmt.Errorf("failed to add vpp_apps_teams_id to policies: %w", err)
}

return nil
}

func Down_20250124194347(tx *sql.Tx) error {
if _, err := tx.Exec(`
ALTER TABLE software_titles
DROP COLUMN coalesce_bundle_name
`); err != nil {
return fmt.Errorf("failed to remove generated column: %w", err)
}

if _, err := tx.Exec(`
ALTER TABLE software_titles
DROP INDEX idx_sw_titles,
ADD UNIQUE KEY idx_sw_titles (name, source, browser);
`); err != nil {
return fmt.Errorf("failed to add vpp_apps_teams_id to policies: %w", err)
}

return nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package tables

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestUp_20250124194347(t *testing.T) {
db := applyUpToPrev(t)

var softwareTitles []struct {
ColumnName string `db:"COLUMN_NAME"`
}

sel := `SELECT COLUMN_NAME
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'software_titles'
AND index_name = 'idx_sw_titles'
ORDER BY seq_in_index;`

err := db.Select(&softwareTitles, sel)
if err != nil {
t.Fatalf("Failed to get index information: %v", err)
}
expected := []struct {
ColumnName string `db:"COLUMN_NAME"`
}{
{ColumnName: "name"},
{ColumnName: "source"},
{ColumnName: "browser"},
}
require.Equal(t, expected, softwareTitles)

applyNext(t, db)

err = db.Select(&softwareTitles, sel)
if err != nil {
t.Fatalf("Failed to get index information: %v", err)
}
expected = []struct {
ColumnName string `db:"COLUMN_NAME"`
}{
{ColumnName: "coalesce_bundle_name"},
{ColumnName: "source"},
{ColumnName: "browser"},
}
require.Equal(t, expected, softwareTitles)
}
7 changes: 4 additions & 3 deletions server/datastore/mysql/schema.sql

Large diffs are not rendered by default.

11 changes: 9 additions & 2 deletions server/datastore/mysql/software.go
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,14 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
argsWithoutBundleIdentifier = append(argsWithoutBundleIdentifier, sw.Name, sw.Source, sw.Browser)
}
// Map software title identifier to software checksums so that we can map checksums to actual titles later.
uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(sw.Name, sw.Source, sw.Browser)] = checksum
uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(
func() string {
if sw.BundleIdentifier != "" {
return sw.BundleIdentifier
}
return sw.Name
}(),
sw.Source, sw.Browser)] = checksum
}

// Get titles for software without bundle_identifier.
Expand Down Expand Up @@ -593,7 +600,7 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
}
// Map software titles to software checksums.
for _, title := range existingSoftwareTitlesForNewSoftwareWithBundleIdentifier {
checksum, ok := uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(title.Name, title.Source, title.Browser)]
checksum, ok := uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(*title.BundleIdentifier, title.Source, title.Browser)]
if ok {
incomingChecksumToTitle[checksum] = title
}
Expand Down
138 changes: 138 additions & 0 deletions server/datastore/mysql/software_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ func TestSoftware(t *testing.T) {
{"SaveHost", testSoftwareSaveHost},
{"CPE", testSoftwareCPE},
{"HostDuplicates", testSoftwareHostDuplicates},
{"DuplicateNameDifferentBundleIdentifier", testSoftwareDuplicateNameDifferentBundleIdentifier},
{"DifferentNameSameBundleIdentifier", testSoftwareDifferentNameSameBundleIdentifier},
{"LoadVulnerabilities", testSoftwareLoadVulnerabilities},
{"ListSoftwareCPEs", testListSoftwareCPEs},
{"NothingChanged", testSoftwareNothingChanged},
Expand Down Expand Up @@ -221,6 +223,142 @@ func testSoftwareCPE(t *testing.T, ds *Datastore) {
require.NoError(t, iterator.Close())
}

func testSoftwareDifferentNameSameBundleIdentifier(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())

incoming := make(map[string]fleet.Software)
sw, err := fleet.SoftwareFromOsqueryRow("GoLand.app", "2024.3", "apps", "", "", "", "", "com.jetbrains.goland", "", "", "")
require.NoError(t, err)
soft2Key := sw.ToUniqueStr()
incoming[soft2Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err := ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err := ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

var software []fleet.Software
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 1)
var softwareTitle []fleet.SoftwareTitle
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 1)

incoming = make(map[string]fleet.Software)
sw, err = fleet.SoftwareFromOsqueryRow("GoLand 2.app", "2024.3", "apps", "", "", "", "", "com.jetbrains.goland", "", "", "")
require.NoError(t, err)
soft3Key := sw.ToUniqueStr()
incoming[soft3Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err = ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err = ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 2)
for _, s := range software {
require.NotEmpty(t, s.TitleID)
}

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 1)
}

func testSoftwareDuplicateNameDifferentBundleIdentifier(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())

incoming := make(map[string]fleet.Software)
sw, err := fleet.SoftwareFromOsqueryRow("a", "0.0.1", "chrome_extension", "", "", "", "", "bundle_id1", "", "", "")
require.NoError(t, err)
soft2Key := sw.ToUniqueStr()
incoming[soft2Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err := ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err := ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

var software []fleet.Software
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 1)
var softwareTitle []fleet.SoftwareTitle
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 1)

incoming = make(map[string]fleet.Software)
sw, err = fleet.SoftwareFromOsqueryRow("a", "0.0.1", "chrome_extension", "", "", "", "", "bundle_id2", "", "", "")
require.NoError(t, err)
soft3Key := sw.ToUniqueStr()
incoming[soft3Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err = ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err = ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 2)
for _, s := range software {
require.NotEmpty(t, s.TitleID)
}

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 2)
}

func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())

Expand Down

0 comments on commit b0457a1

Please sign in to comment.