feat: new migrations system (#876)

* feat: new migration system

* use newFuncMigration

* database version -> database schema version

* column name

* use path instead of filepath for goembed

* simplified migrations, added backwards compatible migrations

go.mod

@@ -5,6 +5,7 @@ go 1.22.2
 
 require (
 	github.com/PuerkitoBio/goquery v1.9.1
+	github.com/blang/semver v3.5.1+incompatible
 	github.com/disintegration/imaging v1.6.2
 	github.com/fatih/color v1.16.0
 	github.com/gin-contrib/requestid v1.0.0
@@ -16,7 +17,6 @@ require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/gofrs/uuid/v5 v5.1.0
 	github.com/golang-jwt/jwt/v5 v5.2.1
-	github.com/golang-migrate/migrate/v4 v4.17.0
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/lib/pq v1.10.9
@@ -28,6 +28,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/afero v1.11.0
 	github.com/spf13/cobra v1.8.0
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
@@ -62,15 +63,11 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
-	github.com/knz/go-libedit v1.10.1 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -83,33 +80,21 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tdewolff/parse v2.3.4+incompatible // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/vincent-petithory/dataurl v1.0.0 // indirect
 	go.etcd.io/bbolt v1.3.9 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.3.0 // indirect
-	modernc.org/cc/v3 v3.41.0 // indirect
-	modernc.org/cc/v4 v4.20.0 // indirect
-	modernc.org/ccgo/v3 v3.17.0 // indirect
-	modernc.org/ccgo/v4 v4.16.0 // indirect
-	modernc.org/gc/v2 v2.4.1 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 	modernc.org/libc v1.49.3 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
 	modernc.org/memory v1.8.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sortutil v1.2.0 // indirect
 	modernc.org/strutil v1.2.0 // indirect
 	modernc.org/token v1.1.0 // indirect
 )

internal/cmd/root.go

@@ -91,7 +91,7 @@ func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *depen
 	}
 
 	// Migrate
-	if err := db.Migrate(); err != nil {
+	if err := db.Migrate(ctx); err != nil {
 		logger.WithError(err).Fatalf("Error running migration")
 	}
 

internal/database/database.go

@@ -2,7 +2,6 @@ package database
 
 import (
 	"context"
-	"embed"
 	"fmt"
 	"log"
 	"net/url"
@@ -13,9 +12,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-//go:embed migrations/*
-var migrations embed.FS
-
 // OrderMethod is the order method for getting bookmarks
 type OrderMethod int
 
@@ -68,8 +64,17 @@ func Connect(ctx context.Context, dbURL string) (DB, error) {
 
 // DB is interface for accessing and manipulating data in database.
 type DB interface {
+	// DBx is the underlying sqlx.DB
+	DBx() sqlx.DB
+
 	// Migrate runs migrations for this database
-	Migrate() error
+	Migrate(ctx context.Context) error
+
+	// GetDatabaseSchemaVersion gets the version of the database
+	GetDatabaseSchemaVersion(ctx context.Context) (string, error)
+
+	// SetDatabaseSchemaVersion sets the version of the database
+	SetDatabaseSchemaVersion(ctx context.Context, version string) error
 
 	// SaveBookmarks saves bookmarks data to database.
 	SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error)

internal/database/migrations.go

@@ -0,0 +1,97 @@
+package database
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"fmt"
+	"path"
+
+	"github.com/blang/semver"
+)
+
+//go:embed migrations/*
+var migrationFiles embed.FS
+
+type migration struct {
+	fromVersion   semver.Version
+	toVersion     semver.Version
+	migrationFunc func(db *sql.DB) error
+}
+
+// txFunc is a function that runs in a transaction.
+type txFn func(tx *sql.Tx) error
+
+// runInTransaction runs the given function in a transaction.
+func runInTransaction(db *sql.DB, fn txFn) error {
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("failed to start transaction: %w", err)
+	}
+	defer tx.Rollback()
+
+	if err := fn(tx); err != nil {
+		return fmt.Errorf("failed to run transaction: %w", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("failed to commit transaction: %w", err)
+	}
+
+	return nil
+}
+
+// newFuncMigration creates a new migration from a function.
+func newFuncMigration(fromVersion, toVersion string, migrationFunc func(db *sql.DB) error) migration {
+	return migration{
+		fromVersion:   semver.MustParse(fromVersion),
+		toVersion:     semver.MustParse(toVersion),
+		migrationFunc: migrationFunc,
+	}
+}
+
+// newFileMigration creates a new migration from a file.
+func newFileMigration(fromVersion, toVersion, filename string) migration {
+	return newFuncMigration(fromVersion, toVersion, func(db *sql.DB) error {
+		return runInTransaction(db, func(tx *sql.Tx) error {
+			migrationSQL, err := migrationFiles.ReadFile(path.Join("migrations", filename+".up.sql"))
+			if err != nil {
+				return fmt.Errorf("failed to read migration file: %w", err)
+			}
+
+			if _, err := tx.Exec(string(migrationSQL)); err != nil {
+				return fmt.Errorf("failed to execute migration %s to %s: %w", fromVersion, toVersion, err)
+			}
+			return nil
+		})
+	})
+}
+
+// runMigrations runs the given migrations.
+func runMigrations(ctx context.Context, db DB, migrations []migration) error {
+	currentVersion := semver.Version{}
+
+	// Get current database version
+	dbVersion, err := db.GetDatabaseSchemaVersion(ctx)
+	if err == nil && dbVersion != "" {
+		currentVersion = semver.MustParse(dbVersion)
+	}
+
+	for _, migration := range migrations {
+		if !currentVersion.EQ(migration.fromVersion) {
+			continue
+		}
+
+		if err := migration.migrationFunc(db.DBx().DB); err != nil {
+			return fmt.Errorf("failed to run migration from %s to %s: %w", migration.fromVersion, migration.toVersion, err)
+		}
+
+		currentVersion = migration.toVersion
+
+		if err := db.SetDatabaseSchemaVersion(ctx, currentVersion.String()); err != nil {
+			return fmt.Errorf("failed to store database version %s from %s to %s: %w", currentVersion.String(), migration.fromVersion, migration.toVersion, err)
+		}
+	}
+
+	return nil
+}

internal/database/migrations/mysql/0000_system_create.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE IF NOT EXISTS shiori_system(
+    database_schema_version VARCHAR(12) NOT NULL DEFAULT '0.0.0'
+);

internal/database/migrations/mysql/0000_system_insert.up.sql

@@ -0,0 +1 @@
+INSERT INTO shiori_system(database_schema_version) VALUES('0.0.0');

internal/database/migrations/mysql/0001_initial.up.sql → internal/database/migrations/mysql/0001_initial_account.up.sql

@@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS account(
 		username VARCHAR(250)   NOT NULL,
 		password BINARY(80)     NOT NULL,
 		owner    TINYINT(1)     NOT NULL DEFAULT '0',
+		config   JSON           NOT NULL DEFAULT '{}',
 		PRIMARY KEY (id),
 		UNIQUE KEY account_username_UNIQUE (username))
 		CHARACTER SET utf8mb4;

internal/database/migrations/mysql/0002_initial_bookmark.up.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS bookmark(
+		id          INT(11)    NOT NULL AUTO_INCREMENT,
+		url         TEXT       NOT NULL,
+		title       TEXT       NOT NULL,
+		excerpt     TEXT       NOT NULL DEFAULT (''),
+		author      TEXT       NOT NULL DEFAULT (''),
+		public      BOOLEAN    NOT NULL DEFAULT 0,
+		content     MEDIUMTEXT NOT NULL DEFAULT (''),
+		html        MEDIUMTEXT NOT NULL DEFAULT (''),
+		modified    TIMESTAMP  NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+		has_content BOOLEAN    NOT NULL DEFAULT 0,
+		PRIMARY KEY(id),
+		UNIQUE KEY bookmark_url_UNIQUE (url(255)),
+		FULLTEXT (title, excerpt, content))
+		CHARACTER SET utf8mb4;

internal/database/migrations/postgres/0000_system.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS shiori_system(
+    database_schema_version TEXT NOT NULL DEFAULT '0.0.0'
+);
+
+INSERT INTO shiori_system(database_schema_version) VALUES('0.0.0');

internal/database/migrations/postgres/0001_initial.up.sql

@@ -1,21 +1,23 @@
 CREATE TABLE IF NOT EXISTS account(
 		id       SERIAL,
 		username VARCHAR(250) NOT NULL,
-		password BYTEA    NOT NULL,
-		owner    BOOLEAN  NOT NULL DEFAULT FALSE,
+		password BYTEA        NOT NULL,
+		owner    BOOLEAN      NOT NULL DEFAULT FALSE,
+		config   JSONB        NOT NULL DEFAULT '{}',
 		PRIMARY KEY (id),
 		CONSTRAINT account_username_UNIQUE UNIQUE (username));
 
 CREATE TABLE IF NOT EXISTS bookmark(
-		id       SERIAL,
-		url      TEXT       NOT NULL,
-		title    TEXT       NOT NULL,
-		excerpt  TEXT       NOT NULL DEFAULT '',
-		author   TEXT       NOT NULL DEFAULT '',
-		public   SMALLINT   NOT NULL DEFAULT 0,
-		content  TEXT       NOT NULL DEFAULT '',
-		html     TEXT       NOT NULL DEFAULT '',
-		modified TIMESTAMP(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+		id          SERIAL,
+		url         TEXT       NOT NULL,
+		title       TEXT       NOT NULL,
+		excerpt     TEXT       NOT NULL DEFAULT '',
+		author      TEXT       NOT NULL DEFAULT '',
+		public      SMALLINT   NOT NULL DEFAULT 0,
+		content     TEXT       NOT NULL DEFAULT '',
+		html        TEXT       NOT NULL DEFAULT '',
+		modified    TIMESTAMP(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+		has_content BOOLEAN NOT NULL DEFAULT FALSE,
 		PRIMARY KEY(id),
 		CONSTRAINT bookmark_url_UNIQUE UNIQUE (url));
 

internal/database/migrations/sqlite/0000_system.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS shiori_system(
+    database_schema_version TEXT NOT NULL DEFAULT '0.0.0'
+);
+
+INSERT INTO shiori_system(database_schema_version) VALUES('0.0.0');

internal/database/migrations/sqlite/0001_initial.up.sql

@@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS account(
     username TEXT NOT NULL,
     password TEXT NOT NULL,
     owner INTEGER NOT NULL DEFAULT 0,
+    config JSON NOT NULL DEFAULT '{}',
     CONSTRAINT account_PK PRIMARY KEY(id),
     CONSTRAINT account_username_UNIQUE UNIQUE(username)
 );
@@ -15,6 +16,7 @@ CREATE TABLE IF NOT EXISTS bookmark(
     author TEXT NOT NULL DEFAULT "",
     public INTEGER NOT NULL DEFAULT 0,
     modified TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    has_content BOOLEAN DEFAULT FALSE NOT NULL,
     CONSTRAINT bookmark_PK PRIMARY KEY(id),
     CONSTRAINT bookmark_url_UNIQUE UNIQUE(url)
 );

internal/database/migrations/sqlite/0002_denormalize_content.up.sql

@@ -1,6 +1,3 @@
-ALTER TABLE bookmark
-    ADD has_content BOOLEAN DEFAULT FALSE NOT NULL;
-
 UPDATE bookmark
 SET has_content = bc.has_content FROM (SELECT docid, content <> '' AS has_content FROM bookmark_content) AS bc
 WHERE bookmark.id = bc.docid;