Skip to content

Commit

Permalink
feat: scylladb database provider implementation (#320)
Browse files Browse the repository at this point in the history
  • Loading branch information
DanielHe4rt authored Nov 20, 2024
1 parent fca0244 commit ff91dce
Show file tree
Hide file tree
Showing 16 changed files with 620 additions and 3 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/generate-linter-core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
strategy:
matrix:
framework: [chi, gin, fiber, gorilla/mux, httprouter, standard-library, echo]
driver: [mysql, postgres, sqlite, mongo, redis, none]
driver: [mysql, postgres, sqlite, mongo, redis, scylla, none]
git: [commit, stage, skip]

runs-on: ubuntu-latest
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/testcontainers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
strategy:
matrix:
driver:
[mysql, postgres, mongo, redis]
[mysql, postgres, mongo, redis, scylla]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ Choose from a variety of supported database drivers:
- [Sqlite](https://github.com/mattn/go-sqlite3)
- [Mongo](https://go.mongodb.org/mongo-driver)
- [Redis](https://github.com/redis/go-redis)
- [ScyllaDB GoCQL](https://github.com/scylladb/gocql)

<a id="advanced-features"></a>

Expand Down
3 changes: 2 additions & 1 deletion cmd/flags/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ const (
Sqlite Database = "sqlite"
Mongo Database = "mongo"
Redis Database = "redis"
Scylla Database = "scylla"
None Database = "none"
)

var AllowedDBDrivers = []string{string(MySql), string(Postgres), string(Sqlite), string(Mongo), string(Redis), string(None)}
var AllowedDBDrivers = []string{string(MySql), string(Postgres), string(Sqlite), string(Mongo), string(Redis), string(Scylla), string(None)}

func (f Database) String() string {
return string(f)
Expand Down
21 changes: 21 additions & 0 deletions cmd/program/program.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ var (
sqliteDriver = []string{"github.com/mattn/go-sqlite3"}
redisDriver = []string{"github.com/redis/go-redis/v9"}
mongoDriver = []string{"go.mongodb.org/mongo-driver"}
gocqlDriver = []string{"github.com/gocql/gocql"}
scyllaDriver = "github.com/scylladb/[email protected]" // Replacement for GoCQL

godotenvPackage = []string{"github.com/joho/godotenv"}
templPackage = []string{"github.com/a-h/templ"}
Expand Down Expand Up @@ -206,6 +208,11 @@ func (p *Project) createDBDriverMap() {
packageName: redisDriver,
templater: dbdriver.RedisTemplate{},
}

p.DBDriverMap[flags.Scylla] = Driver{
packageName: gocqlDriver,
templater: dbdriver.ScyllaTemplate{},
}
}

func (p *Project) createDockerMap() {
Expand All @@ -227,6 +234,10 @@ func (p *Project) createDockerMap() {
packageName: []string{},
templater: docker.RedisDockerTemplate{},
}
p.DockerMap[flags.Scylla] = Docker{
packageName: []string{},
templater: docker.ScyllaDockerTemplate{},
}
}

// CreateMainFile creates the project folders and files,
Expand Down Expand Up @@ -335,12 +346,22 @@ func (p *Project) CreateMainFile() error {

// Install the godotenv package
err = utils.GoGetPackage(projectPath, godotenvPackage)

if err != nil {
log.Printf("Could not install go dependency %v\n", err)

return err
}

if p.DBDriver == flags.Scylla {
replace := fmt.Sprintf("%s=%s", gocqlDriver[0], scyllaDriver)
err = utils.GoModReplace(projectPath, replace)
if err != nil {
log.Printf("Could not replace go dependency %v\n", err)
return err
}
}

err = p.CreatePath(cmdApiPath, projectPath)
if err != nil {
log.Printf("Error creating path: %s", projectPath)
Expand Down
3 changes: 3 additions & 0 deletions cmd/steps/steps.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ func InitSteps(projectType flags.Framework, databaseType flags.Database) *Steps
{
Title: "Redis",
Desc: "Redis driver for Go."},
{
Title: "Scylla",
Desc: "ScyllaDB Enhanced driver from GoCQL."},
{
Title: "None",
Desc: "Choose this option if you don't wish to install a specific database driver."},
Expand Down
10 changes: 10 additions & 0 deletions cmd/template/dbdriver/files/env/scylla.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{{- if .AdvancedOptions.docker }}
# BLUEPRINT_DB_HOSTS=scylla_bp:9042 # ScyllaDB default port
BLUEPRINT_DB_HOSTS=scylla_bp:19042 # ScyllaDB Shard-Aware port
{{- else }}
# BLUEPRINT_DB_HOSTS=localhost:9042 # ScyllaDB default port
BLUEPRINT_DB_HOSTS=localhost:19042 # ScyllaDB Shard-Aware port
{{- end }}
BLUEPRINT_DB_CONSISTENCY="LOCAL_QUORUM"
# BLUEPRINT_DB_USERNAME=
# BLUEPRINT_DB_PASSWORD=
165 changes: 165 additions & 0 deletions cmd/template/dbdriver/files/service/scylla.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package database

import (
"context"
"fmt"
"log"
"os"
"strconv"
"strings"
"time"

"github.com/gocql/gocql"
_ "github.com/joho/godotenv/autoload"
)

// Service defines the interface for health checks.
type Service interface {
Health() map[string]string
Close() error
}

// service implements the Service interface.
type service struct {
Session *gocql.Session
}

// Environment variables for ScyllaDB connection.
var (
hosts = os.Getenv("BLUEPRINT_DB_HOSTS") // Comma-separated list of hosts:port
username = os.Getenv("BLUEPRINT_DB_USERNAME") // Username for authentication
password = os.Getenv("BLUEPRINT_DB_PASSWORD") // Password for authentication
consistencyLevel = os.Getenv("BLUEPRINT_DB_CONSISTENCY") // Consistency level
)

// New initializes a new Service with a ScyllaDB Session.
func New() Service {
cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy())

// Set authentication if provided
if username != "" && password != "" {
cluster.Authenticator = gocql.PasswordAuthenticator{
Username: username,
Password: password,
}
}

// Set consistency level if provided
if consistencyLevel != "" {
if cl, err := parseConsistency(consistencyLevel); err == nil {
cluster.Consistency = cl
} else {
log.Printf("Invalid SCYLLA_DB_CONSISTENCY '%s', using default: %v", consistencyLevel, err)
}
}

// Create Session
session, err := cluster.CreateSession()
if err != nil {
log.Fatalf("Failed to connect to ScyllaDB cluster: %v", err)
}

s := &service{Session: session}
return s
}

// parseConsistency converts a string to a gocql.Consistency value.
func parseConsistency(cons string) (gocql.Consistency, error) {
consistencyMap := map[string]gocql.Consistency{
"ANY": gocql.Any,
"ONE": gocql.One,
"TWO": gocql.Two,
"THREE": gocql.Three,
"QUORUM": gocql.Quorum,
"ALL": gocql.All,
"LOCAL_ONE": gocql.LocalOne,
"LOCAL_QUORUM": gocql.LocalQuorum,
"EACH_QUORUM": gocql.EachQuorum,
}

if consistency, ok := consistencyMap[strings.ToUpper(cons)]; ok {
return consistency, nil
}
return gocql.LocalQuorum, fmt.Errorf("unknown consistency level: %s", cons)
}

// Health returns the health status and statistics of the ScyllaDB cluster.
func (s *service) Health() map[string]string {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

stats := make(map[string]string)

// Check ScyllaDB health and populate the stats map
startedAt := time.Now()

// Execute a simple query to check connectivity
query := "SELECT now() FROM system.local"
iter := s.Session.Query(query).WithContext(ctx).Iter()
var currentTime time.Time
if !iter.Scan(&currentTime) {
if err := iter.Close(); err != nil {
stats["status"] = "down"
stats["message"] = fmt.Sprintf("Failed to execute query: %v", err)
return stats
}
}
if err := iter.Close(); err != nil {
stats["status"] = "down"
stats["message"] = fmt.Sprintf("Error during query execution: %v", err)
return stats
}

// ScyllaDB is up
stats["status"] = "up"
stats["message"] = "It's healthy"
stats["scylla_current_time"] = currentTime.String()

// Retrieve cluster information
// Get keyspace information
getKeyspacesQuery := "SELECT keyspace_name FROM system_schema.keyspaces"
keyspacesIterator := s.Session.Query(getKeyspacesQuery).Iter()

stats["scylla_keyspaces"] = strconv.Itoa(keyspacesIterator.NumRows())
if err := keyspacesIterator.Close(); err != nil {
log.Fatalf("Failed to close keyspaces iterator: %v", err)
}

// Get cluster information
var currentDatacenter string
var currentHostStatus bool

var clusterNodesUp uint
var clusterNodesDown uint
var clusterSize uint

clusterNodesIterator := s.Session.Query("SELECT dc, up FROM system.cluster_status").Iter()
for clusterNodesIterator.Scan(&currentDatacenter, &currentHostStatus) {
clusterSize++
if currentHostStatus {
clusterNodesUp++
} else {
clusterNodesDown++
}
}

if err := clusterNodesIterator.Close(); err != nil {
log.Fatalf("Failed to close cluster nodes iterator: %v", err)
}

stats["scylla_cluster_size"] = strconv.Itoa(int(clusterSize))
stats["scylla_cluster_nodes_up"] = strconv.Itoa(int(clusterNodesUp))
stats["scylla_cluster_nodes_down"] = strconv.Itoa(int(clusterNodesDown))
stats["scylla_current_datacenter"] = currentDatacenter

// Calculate the time taken to perform the health check
stats["scylla_health_check_duration"] = time.Since(startedAt).String()
return stats
}

// Close gracefully closes the ScyllaDB Session.
func (s *service) Close() error {
s.Session.Close()
return nil
}
Loading

1 comment on commit ff91dce

@pudds03-ops
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can I get the whole code process

Please sign in to comment.