From dcc0f67f7822c4656ebcbaeb73cc36587fdd48a9 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Fri, 5 Jul 2024 23:45:39 -0400 Subject: [PATCH] Switch to use server side schema description for KeyspaceMetadata.ToCQL No need to keep client-side logic which needs to be updated every time there is changes on server-side. --- metadata_scylla.go | 40 +++++++++++++++++++++++++- metadata_scylla_test.go | 2 +- recreate.go | 9 +++++- recreate_test.go | 64 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/metadata_scylla.go b/metadata_scylla.go index 7996a254e..56aca0512 100644 --- a/metadata_scylla.go +++ b/metadata_scylla.go @@ -25,6 +25,7 @@ type KeyspaceMetadata struct { Types map[string]*TypeMetadata Indexes map[string]*IndexMetadata Views map[string]*ViewMetadata + CreateStmts string } // schema metadata for a table (a.k.a. column family) @@ -358,8 +359,13 @@ func (s *schemaDescriber) refreshSchema(keyspaceName string) error { return err } + createStmts, err := getCreateStatements(s.session, keyspaceName) + if err != nil { + return err + } + // organize the schema data - compileMetadata(keyspace, tables, columns, functions, aggregates, types, indexes, views) + compileMetadata(keyspace, tables, columns, functions, aggregates, types, indexes, views, createStmts) // update the cache s.cache[keyspaceName] = keyspace @@ -381,6 +387,7 @@ func compileMetadata( types []TypeMetadata, indexes []IndexMetadata, views []ViewMetadata, + createStmts []byte, ) { keyspace.Tables = make(map[string]*TableMetadata) for i := range tables { @@ -452,6 +459,8 @@ func compileMetadata( v := &views[i] v.PartitionKey, v.ClusteringColumns, v.OrderedColumns = compileColumns(v.Columns, v.OrderedColumns) } + + keyspace.CreateStmts = string(createStmts) } func compileColumns(columns map[string]*ColumnMetadata, orderedColumns []string) ( @@ -744,6 +753,35 @@ func getIndexMetadata(session *Session, keyspaceName string) ([]IndexMetadata, e return indexes, nil } +// get create statements for the keyspace +func getCreateStatements(session *Session, keyspaceName string) ([]byte, error) { + if !session.useSystemSchema { + return nil, nil + } + iter := session.control.query(fmt.Sprintf(`DESCRIBE KEYSPACE %s WITH INTERNALS`, keyspaceName)) + + var createStatements []string + + var stmt string + for iter.Scan(nil, nil, nil, &stmt) { + if stmt == "" { + continue + } + createStatements = append(createStatements, stmt) + } + + if err := iter.Close(); err != nil { + if errFrame, ok := err.(errorFrame); ok && errFrame.code == ErrCodeSyntax { + // DESCRIBE KEYSPACE is not supported on older versions of Cassandra and Scylla + // For such case schema statement is going to be recreated on the client side + return nil, nil + } + return nil, fmt.Errorf("error querying keyspace schema: %v", err) + } + + return []byte(strings.Join(createStatements, "\n")), nil +} + // query for view metadata in the system_schema.views func getViewMetadata(session *Session, keyspaceName string) ([]ViewMetadata, error) { if !session.useSystemSchema { diff --git a/metadata_scylla_test.go b/metadata_scylla_test.go index ae541ed9f..a1838302c 100644 --- a/metadata_scylla_test.go +++ b/metadata_scylla_test.go @@ -113,7 +113,7 @@ func TestCompileMetadata(t *testing.T) { ViewName: "sec_idx_index", }, } - compileMetadata(keyspace, tables, columns, nil, nil, nil, indexes, views) + compileMetadata(keyspace, tables, columns, nil, nil, nil, indexes, views, nil) assertKeyspaceMetadata( t, keyspace, diff --git a/recreate.go b/recreate.go index 738d898a1..5ff9f8ad7 100644 --- a/recreate.go +++ b/recreate.go @@ -20,6 +20,12 @@ import ( // user defined types, tables, indexes, functions, aggregates and views associated // with this keyspace. func (km *KeyspaceMetadata) ToCQL() (string, error) { + // Be aware that `CreateStmts` is not only a cache for ToCQL, + // but it also can be populated from response to `DESCRIBE KEYSPACE %s WITH INTERNALS` + if len(km.CreateStmts) != 0 { + return km.CreateStmts, nil + } + var sb strings.Builder if err := km.keyspaceToCQL(&sb); err != nil { @@ -63,7 +69,8 @@ func (km *KeyspaceMetadata) ToCQL() (string, error) { } } - return sb.String(), nil + km.CreateStmts = sb.String() + return km.CreateStmts, nil } func (km *KeyspaceMetadata) typesSortedTopologically() []*TypeMetadata { diff --git a/recreate_test.go b/recreate_test.go index b1df27690..88bbe0f84 100644 --- a/recreate_test.go +++ b/recreate_test.go @@ -7,7 +7,10 @@ package gocql import ( "encoding/json" + "flag" + "fmt" "io/ioutil" + "regexp" "sort" "strings" "testing" @@ -15,10 +18,14 @@ import ( "github.com/google/go-cmp/cmp" ) +var updateGolden = flag.Bool("update-golden", false, "update golden files") + func TestRecreateSchema(t *testing.T) { session := createSessionFromCluster(createCluster(), t) defer session.Close() + getStmtFromCluster := isDescribeKeyspaceSupported(t, session) + tcs := []struct { Name string Keyspace string @@ -97,13 +104,30 @@ func TestRecreateSchema(t *testing.T) { t.Fatal("recreate schema", err) } - golden, err := ioutil.ReadFile(test.Golden) - if err != nil { - t.Fatal(err) + dump = trimSchema(dump) + + var golden []byte + if getStmtFromCluster { + golden, err = getCreateStatements(session, test.Keyspace) + if err != nil { + t.Fatal(err) + } + golden = []byte(trimSchema(string(golden))) + } else { + if *updateGolden { + if err := ioutil.WriteFile(test.Golden, []byte(dump), 0644); err != nil { + t.Fatal(err) + } + } + golden, err = ioutil.ReadFile(test.Golden) + if err != nil { + t.Fatal(err) + } + golden = []byte(trimSchema(string(golden))) } - goldenQueries := sortQueries(strings.Split(string(golden), ";")) - dumpQueries := sortQueries(strings.Split(dump, ";")) + goldenQueries := trimQueries(sortQueries(strings.Split(string(golden), ";"))) + dumpQueries := trimQueries(sortQueries(strings.Split(dump, ";"))) if len(goldenQueries) != len(dumpQueries) { t.Errorf("Expected len(dumpQueries) to be %d, got %d", len(goldenQueries), len(dumpQueries)) @@ -139,7 +163,9 @@ func TestRecreateSchema(t *testing.T) { t.Fatal("recreate schema", err) } - secondDumpQueries := sortQueries(strings.Split(secondDump, ";")) + secondDump = trimSchema(secondDump) + + secondDumpQueries := trimQueries(sortQueries(strings.Split(secondDump, ";"))) if !cmp.Equal(secondDumpQueries, dumpQueries) { t.Errorf("first dump and second one differs: %s", cmp.Diff(secondDumpQueries, dumpQueries)) @@ -148,6 +174,21 @@ func TestRecreateSchema(t *testing.T) { } } +func isDescribeKeyspaceSupported(t *testing.T, s *Session) bool { + t.Helper() + + err := s.control.query(fmt.Sprintf(`DESCRIBE KEYSPACE system WITH INTERNALS`)).Close() + if err != nil { + if errFrame, ok := err.(errorFrame); ok && errFrame.code == ErrCodeSyntax { + // DESCRIBE KEYSPACE is not supported on older versions of Cassandra and Scylla + // For such case schema statement is going to be recreated on the client side + return false + } + t.Fatalf("error querying keyspace schema: %v", err) + } + return true +} + func TestScyllaEncryptionOptionsUnmarshaller(t *testing.T) { const ( input = "testdata/recreate/scylla_encryption_options.bin" @@ -198,9 +239,20 @@ func trimQueries(in []string) []string { queries := in[:0] for _, q := range in { q = strings.TrimSpace(q) + if q == "" { + continue + } if len(q) != 0 { queries = append(queries, q) } } return queries } + +var schemaVersion = regexp.MustCompile(` WITH ID = [0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}[ \t\n]+AND`) + +func trimSchema(s string) string { + // Remove temporary items from the scheme, in particular schema version: + // ) WITH ID = cf0364d0-3b85-11ef-b79d-80a2ee1928c0 + return strings.ReplaceAll(schemaVersion.ReplaceAllString(s, " WITH"), "\n\n", "\n") +}