refactor: allow specific sqlite engine for OpenBSD (#780)

* remove dependencies that use syscall.Syscall*

OpenBSD will be removing direct access to `syscall(2)` soon.
Shiori will stop working because of this, as some of its dependencies
rely heavily on the use of `syscall.Syscall*`, which ends up using
`syscall(2)`.  This commit removes those dependencies by reverting back
to use github.com/mattn/go-sqlite3 instead of modernc.org/sqlite to deal
with the sqlite database backend.

* add ~emersion/go-sqlite3-fts5

* enable cgo to check ci

* fts5 build flag

* split sqlite logic using build flags

* disable cgo again

* added ci test for bsd systems

* remove openbsd ci

* Revert "remove openbsd ci"

This reverts commit f394148.

* fix makefile go_test_flags

---------

Co-authored-by: Felipe M <[email protected]>
Co-authored-by: Felipe Martin <[email protected]>

.github/workflows/_test.yml

@@ -89,8 +89,41 @@ jobs:
           golangci-lint.cache-{interval_number}-
           golangci-lint.cache-
 
-    - run: make unittest GO_TEST_FLAGS="-tags test_sqlite_only"
+    - run: make unittest GO_TEST_FLAGS="-tags test_sqlite_only -race -v -count=1"
       env:
         CGO_ENABLED: 1 # go test -race requires cgo
 
     - run: go build -tags osusergo,netgo -ldflags="-s -w -X main.version=$(git describe --tags) -X main.date=$(date --iso-8601=seconds)"
+
+  # Please note BSD support is offered on a best-effort basis, this check is not blocking but for us to be aware of issues.
+  # This test also does not take into consideration the go version specified in the go.mod file and just uses the
+  # latest version available in the openbsd package repository.
+  test-bsd:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os:
+          - name: openbsd
+            architecture: x86-64
+            version: '7.5'
+
+          - name: openbsd
+            architecture: arm64
+            version: '7.5'
+    steps:
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+
+      - name: Test on ${{ matrix.os.name }}
+        uses: cross-platform-actions/action@b2e15da1e667187766fff4945d20b98ac7055576 # v0.24.0
+        with:
+          environment_variables: GO_VERSION
+          operating_system: ${{ matrix.os.name }}
+          architecture: ${{ matrix.os.architecture }}
+          version: ${{ matrix.os.version }}
+          shell: bash
+          memory: 1G
+          cpu_count: 1
+          run: |
+            sudo pkg_add -u
+            sudo pkg_add gmake git go
+            gmake unittest GO_TEST_FLAGS="-tags test_sqlite_only -v -count=1"

Makefile

@@ -4,17 +4,17 @@ BASH ?= $(shell command -v bash 2> /dev/null)
 # Development
 SHIORI_DIR ?= dev-data
 
-# Testing
-override GO_TEST_FLAGS += -v -race -count=1 -covermode=atomic -coverprofile=coverage.out
-GOTESTFMT_FLAGS ?=
-
 # Build
 CGO_ENABLED ?= 0
 BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S)
 BUILD_HASH := $(shell git describe --tags)
-BUILD_TAGS ?= osusergo,netgo
+BUILD_TAGS ?= osusergo,netgo,fts5
 LDFLAGS += -s -w -X main.version=$(BUILD_HASH) -X main.date=$(BUILD_TIME)
 
+# Testing
+GO_TEST_FLAGS ?= -v -race -count=1 -tags $(BUILD_TAGS) -covermode=atomic -coverprofile=coverage.out
+GOTESTFMT_FLAGS ?=
+
 # Development
 GIN_MODE ?= debug
 SHIORI_DEVELOPMENT ?= true

go.mod

@@ -4,6 +4,7 @@ module github.com/go-shiori/shiori
 go 1.22.2
 
 require (
+	git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc
 	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/disintegration/imaging v1.6.2
@@ -20,6 +21,7 @@ require (
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/lib/pq v1.10.9
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/muesli/go-app-paths v0.2.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1

go.sum

@@ -1,5 +1,7 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc h1:+y3OijpLl4rgbFsqMBmYUTCsGCkxQUWpWaqfS8j9Ygc=
+git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc/go.mod h1:PCl1xjl7iC6x35TKKubKRyo/3TT0dGI66jyNI6vmYnU=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=

internal/database/sqlite.go

@@ -78,18 +78,6 @@ type tagContent struct {
 	model.Tag
 }
 
-// OpenSQLiteDatabase creates and open connection to new SQLite3 database.
-func OpenSQLiteDatabase(ctx context.Context, databasePath string) (sqliteDB *SQLiteDatabase, err error) {
-	// Open database
-	db, err := sqlx.ConnectContext(ctx, "sqlite", databasePath)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	sqliteDB = &SQLiteDatabase{dbbase: dbbase{*db}}
-	return sqliteDB, nil
-}
-
 // DBX returns the underlying sqlx.DB object
 func (db *SQLiteDatabase) DBx() sqlx.DB {
 	return db.DB

internal/database/sqlite_noncgo.go

@@ -0,0 +1,25 @@
+//go:build linux || windows || darwin
+// +build linux windows darwin
+
+package database
+
+import (
+	"context"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/pkg/errors"
+
+	_ "modernc.org/sqlite"
+)
+
+// OpenSQLiteDatabase creates and open connection to new SQLite3 database.
+func OpenSQLiteDatabase(ctx context.Context, databasePath string) (sqliteDB *SQLiteDatabase, err error) {
+	// Open database
+	db, err := sqlx.ConnectContext(ctx, "sqlite", databasePath)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	sqliteDB = &SQLiteDatabase{dbbase: dbbase{*db}}
+	return sqliteDB, nil
+}

internal/database/sqlite_openbsd.go

@@ -0,0 +1,26 @@
+//go:build openbsd
+// +build openbsd
+
+package database
+
+import (
+	"context"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/pkg/errors"
+
+	_ "git.sr.ht/~emersion/go-sqlite3-fts5"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+// OpenSQLiteDatabase creates and open connection to new SQLite3 database.
+func OpenSQLiteDatabase(ctx context.Context, databasePath string) (sqliteDB *SQLiteDatabase, err error) {
+	// Open database
+	db, err := sqlx.ConnectContext(ctx, "sqlite3", databasePath)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	sqliteDB = &SQLiteDatabase{dbbase: dbbase{*db}}
+	return sqliteDB, nil
+}