From 41412d056187676eb4e818c431f2807dbfb7d910 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 29 Jun 2017 15:02:32 +0300 Subject: [PATCH] Implemented client.SyncSchema --- README.md | 23 ++++++----------- client.go | 37 ++++++++++++++++++++++++++ client_it_test.go | 43 ++++++++++++++++++++++++++++++ orm.go | 49 +++++++++++++++++++++++++++++++++++ orm_test.go | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a53e3a8..e60a519 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,9 @@ Go client for Pilosa high performance distributed bitmap index. * **next**: * Supports imports. - * Introducing `client.Schema` which retrieves the schema information from the server side. No need to re-define already existing indexes and frames. + * Introduced schemas. No need to re-define already existing indexes and frames. * **Deprecation** `NewIndex`. Use `schema.Index` instead. + * **Deprecation** `CreateIndex`, `CreateFrame`, `EnsureIndex`, `EnsureFrame`. Use schemas and `client.SyncSchema` instead. * **v0.4.0** (2017-06-09): * Supports Pilosa Server v0.4.0. @@ -63,14 +64,11 @@ schema, err := client.Schema() // Create an Index object myindex, err := schema.Index("myindex", nil) -// Make sure the index exists on the server -err = client.EnsureIndex(myindex) - // Create a Frame object myframe, err := myindex.Frame("myframe", nil) -// Make sure the frame exists on the server -err = client.EnsureFrame(myframe) +// make sure the index and frame exists on the server +err = client.SyncSchema(schema) // Send a SetBit query. PilosaException is thrown if execution of the query fails. err = client.Query(myframe.SetBit(5, 42), nil) @@ -124,7 +122,7 @@ options := &pilosa.IndexOptions{ repository, err := schema.Index("repository", options); ``` -Frames are created with a call to `Frame` function of an index: +Frames definitions are created with a call to `Frame` function of an index: ```go stargazer, err := repository.Frame("stargazer", nil) @@ -273,20 +271,15 @@ options = &pilosa.ClientOptions{ client := pilosa.NewClientWithCluster(cluster, options) ``` -Once you create a client, you can create indexes, frames and then start sending queries. +Once you create a client, you can create indexes, frames or start sending queries. Here is how you would create a index and frame: ```go -// materialize repository index instance initialized before -err := client.CreateIndex(repository) - -// materialize stargazer frame instance initialized before -err :=client.CreateFrame(stargazer) +// materialize repository index definition and stargazer frame definition initialized before +err := client.SyncSchema(schema) ``` -If the index or frame exists on the server, non-nil errors will be returned. You can use `EnsureIndex` and `EnsureFrame` functions to ignore existing indexes and frames. - You can send queries to a Pilosa server using the `Query` function of the `Client` struct: ```go diff --git a/client.go b/client.go index 31d342d..9b2bd02 100644 --- a/client.go +++ b/client.go @@ -170,6 +170,43 @@ func (c *Client) DeleteFrame(frame *Frame) error { return err } +// SyncSchema updates a schema with the indexes and frames on the server and +// creates the indexes and frames in the schema on the server side. +// This function does not delete indexes and the frames on the server side nor in the schema. +func (c *Client) SyncSchema(schema *Schema) error { + var err error + serverSchema, err := c.Schema() + if err != nil { + return err + } + + // find out local - remote schema + diffSchema := schema.diff(serverSchema) + // create the indexes and frames which doesn't exist on the server side + for indexName, index := range diffSchema.indexes { + if _, ok := serverSchema.indexes[indexName]; !ok { + c.EnsureIndex(index) + } + for _, frame := range index.frames { + c.EnsureFrame(frame) + } + } + + // find out remote - local schema + diffSchema = serverSchema.diff(schema) + for indexName, index := range diffSchema.indexes { + if serverIndex, ok := schema.indexes[indexName]; !ok { + schema.indexes[indexName] = index + } else { + for frameName, frame := range index.frames { + serverIndex.frames[frameName] = frame + } + } + } + + return nil +} + // Schema returns the indexes and frames on the server. func (c *Client) Schema() (*Schema, error) { status, err := c.status() diff --git a/client_it_test.go b/client_it_test.go index 7a1bc32..89af83b 100644 --- a/client_it_test.go +++ b/client_it_test.go @@ -499,6 +499,49 @@ func TestSchema(t *testing.T) { } } +func TestSync(t *testing.T) { + client := getClient() + remoteIndex, _ := NewIndex("remote-index-1", nil) + err := client.EnsureIndex(remoteIndex) + if err != nil { + t.Fatal(err) + } + remoteFrame, _ := remoteIndex.Frame("remote-frame-1", nil) + err = client.EnsureFrame(remoteFrame) + if err != nil { + t.Fatal(err) + } + schema1 := NewSchema() + index11, _ := schema1.Index("diff-index1", nil) + index11.Frame("frame1-1", nil) + index11.Frame("frame1-2", nil) + index12, _ := schema1.Index("diff-index2", nil) + index12.Frame("frame2-1", nil) + schema1.Index(remoteIndex.Name(), nil) + + err = client.SyncSchema(schema1) + if err != nil { + t.Fatal(err) + } + client.DeleteIndex(remoteIndex) + client.DeleteIndex(index11) + client.DeleteIndex(index12) +} + +func TestSyncFailure(t *testing.T) { + server := getMockServer(404, []byte("sorry, not found"), -1) + defer server.Close() + uri, err := NewURIFromAddress(server.URL) + if err != nil { + panic(err) + } + client := NewClientWithURI(uri) + err = client.SyncSchema(NewSchema()) + if err == nil { + t.Fatal("should have failed") + } +} + func TestErrorRetrievingSchema(t *testing.T) { server := getMockServer(404, []byte("sorry, not found"), -1) defer server.Close() diff --git a/orm.go b/orm.go index 97d65c0..ac19c85 100644 --- a/orm.go +++ b/orm.go @@ -68,6 +68,31 @@ func (s *Schema) Index(name string, options *IndexOptions) (*Index, error) { return index, nil } +func (s *Schema) diff(other *Schema) *Schema { + result := NewSchema() + for indexName, index := range s.indexes { + if otherIndex, ok := other.indexes[indexName]; !ok { + // if the index doesn't exist in the other schema, simply copy it + result.indexes[indexName] = index.copy() + } else { + // the index exists in the other schema; check the frames + resultIndex, _ := NewIndex(indexName, index.options) + for frameName, frame := range index.frames { + if _, ok := otherIndex.frames[frameName]; !ok { + // the frame doesn't exist in the other schame, copy it + resultIndex.frames[frameName] = frame.copy() + } + } + // check whether we modified result index + if len(resultIndex.frames) > 0 { + // if so, move it to the result + result.indexes[indexName] = resultIndex + } + } + } + return result +} + // PQLQuery is an interface for PQL queries. type PQLQuery interface { Index() *Index @@ -223,6 +248,20 @@ func NewIndex(name string, options *IndexOptions) (*Index, error) { }, nil } +func (d *Index) copy() *Index { + frames := make(map[string]*Frame) + for name, f := range d.frames { + frames[name] = f.copy() + } + index := &Index{ + name: d.name, + frames: frames, + options: &IndexOptions{}, + } + *index.options = *d.options + return index +} + // Name returns the name of this index. func (d *Index) Name() string { return d.name @@ -379,6 +418,16 @@ type Frame struct { options *FrameOptions } +func (f *Frame) copy() *Frame { + frame := &Frame{ + name: f.name, + index: f.index, + options: &FrameOptions{}, + } + *frame.options = *f.options + return frame +} + // Bitmap creates a bitmap query using the row label. // Bitmap retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. // It also retrieves any attributes set on that row or column. diff --git a/orm_test.go b/orm_test.go index ee9c6be..639388b 100644 --- a/orm_test.go +++ b/orm_test.go @@ -33,6 +33,7 @@ package pilosa import ( + "reflect" "testing" "time" ) @@ -47,6 +48,31 @@ var b2 = sampleFrame.Bitmap(20) var b3 = sampleFrame.Bitmap(42) var b4 = collabFrame.Bitmap(2) +func TestSchemaDiff(t *testing.T) { + schema1 := NewSchema() + index11, _ := schema1.Index("diff-index1", nil) + index11.Frame("frame1-1", nil) + index11.Frame("frame1-2", nil) + index12, _ := schema1.Index("diff-index2", nil) + index12.Frame("frame2-1", nil) + + schema2 := NewSchema() + index21, _ := schema2.Index("diff-index1", nil) + index21.Frame("another-frame", nil) + + targetDiff12 := NewSchema() + targetIndex1, _ := targetDiff12.Index("diff-index1", nil) + targetIndex1.Frame("frame1-1", nil) + targetIndex1.Frame("frame1-2", nil) + targetIndex2, _ := targetDiff12.Index("diff-index2", nil) + targetIndex2.Frame("frame2-1", nil) + + diff12 := schema1.diff(schema2) + if !reflect.DeepEqual(targetDiff12, diff12) { + t.Fatalf("The diff must be correctly calculated") + } +} + func TestNewIndex(t *testing.T) { index1, err := schema.Index("index-name", nil) if err != nil { @@ -69,6 +95,28 @@ func TestNewIndexWithInvalidName(t *testing.T) { } } +func TestIndexCopy(t *testing.T) { + indexOptions := &IndexOptions{ + ColumnLabel: "columnlabel", + TimeQuantum: TimeQuantumMonthDay, + } + index, err := schema.Index("my-index-4copy", indexOptions) + if err != nil { + t.Fatal(err) + } + options := &FrameOptions{ + RowLabel: "rowlabel", + } + _, err = index.Frame("my-frame-4copy", options) + if err != nil { + t.Fatal(err) + } + copiedIndex := index.copy() + if !reflect.DeepEqual(index, copiedIndex) { + t.Fatalf("copied index should be equivalent") + } +} + func TestFrame(t *testing.T) { frame1, err := sampleIndex.Frame("nonexistent-frame", nil) if err != nil { @@ -83,6 +131,24 @@ func TestFrame(t *testing.T) { } } +func TestFrameCopy(t *testing.T) { + options := &FrameOptions{ + RowLabel: "rowlabel", + TimeQuantum: TimeQuantumMonthDayHour, + CacheType: CacheTypeRanked, + CacheSize: 123456, + InverseEnabled: true, + } + frame, err := sampleIndex.Frame("my-frame-4copy", options) + if err != nil { + t.Fatal(err) + } + copiedFrame := frame.copy() + if !reflect.DeepEqual(frame, copiedFrame) { + t.Fatalf("copied frame should be equivalent") + } +} + func TestNewFrameWithInvalidName(t *testing.T) { index, err := NewIndex("foo", nil) if err != nil {