Implemented client.SyncSchema

README.md

@@ -14,8 +14,9 @@ Go client for Pilosa high performance distributed bitmap index.
 
 * **next**:
     * Supports imports.
-    * Introducing `client.Schema` which retrieves the schema information from the server side. No need to re-define already existing indexes and frames.
+    * Introduced schemas. No need to re-define already existing indexes and frames.
     * **Deprecation** `NewIndex`. Use `schema.Index` instead.
+    * **Deprecation** `CreateIndex`, `CreateFrame`, `EnsureIndex`, `EnsureFrame`. Use schemas and `client.SyncSchema` instead.
 
 * **v0.4.0** (2017-06-09):
     * Supports Pilosa Server v0.4.0.
@@ -63,14 +64,11 @@ schema, err := client.Schema()
 // Create an Index object
 myindex, err := schema.Index("myindex", nil)
 
-// Make sure the index exists on the server
-err = client.EnsureIndex(myindex)
-
 // Create a Frame object
 myframe, err := myindex.Frame("myframe", nil)
 
-// Make sure the frame exists on the server
-err = client.EnsureFrame(myframe)
+// make sure the index and frame exists on the server
+err = client.SyncSchema(schema)
 
 // Send a SetBit query. PilosaException is thrown if execution of the query fails.
 err = client.Query(myframe.SetBit(5, 42), nil)
@@ -124,7 +122,7 @@ options := &pilosa.IndexOptions{
 repository, err := schema.Index("repository", options);
 ```
 
-Frames are created with a call to `Frame` function of an index:
+Frames definitions are created with a call to `Frame` function of an index:
 
 ```go
 stargazer, err := repository.Frame("stargazer", nil)
@@ -273,20 +271,15 @@ options = &pilosa.ClientOptions{
 client := pilosa.NewClientWithCluster(cluster, options)
 ```
 
-Once you create a client, you can create indexes, frames and then start sending queries.
+Once you create a client, you can create indexes, frames or start sending queries.
 
 Here is how you would create a index and frame:
 
 ```go
-// materialize repository index instance initialized before
-err := client.CreateIndex(repository)
-
-// materialize stargazer frame instance initialized before
-err :=client.CreateFrame(stargazer)
+// materialize repository index definition and stargazer frame definition initialized before
+err := client.SyncSchema(schema)
 ```
 
-If the index or frame exists on the server, non-nil errors will be returned. You can use `EnsureIndex` and `EnsureFrame` functions to ignore existing indexes and frames.
-
 You can send queries to a Pilosa server using the `Query` function of the `Client` struct:
 
 ```go

client.go

@@ -170,6 +170,43 @@ func (c *Client) DeleteFrame(frame *Frame) error {
 	return err
 }
 
+// SyncSchema updates a schema with the indexes and frames on the server and
+// creates the indexes and frames in the schema on the server side.
+// This function does not delete indexes and the frames on the server side nor in the schema.
+func (c *Client) SyncSchema(schema *Schema) error {
+	var err error
+	serverSchema, err := c.Schema()
+	if err != nil {
+		return err
+	}
+
+	// find out local - remote schema
+	diffSchema := schema.diff(serverSchema)
+	// create the indexes and frames which doesn't exist on the server side
+	for indexName, index := range diffSchema.indexes {
+		if _, ok := serverSchema.indexes[indexName]; !ok {
+			c.EnsureIndex(index)
+		}
+		for _, frame := range index.frames {
+			c.EnsureFrame(frame)
+		}
+	}
+
+	// find out remote - local schema
+	diffSchema = serverSchema.diff(schema)
+	for indexName, index := range diffSchema.indexes {
+		if serverIndex, ok := schema.indexes[indexName]; !ok {
+			schema.indexes[indexName] = index
+		} else {
+			for frameName, frame := range index.frames {
+				serverIndex.frames[frameName] = frame
+			}
+		}
+	}
+
+	return nil
+}
+
 // Schema returns the indexes and frames on the server.
 func (c *Client) Schema() (*Schema, error) {
 	status, err := c.status()

client_it_test.go

@@ -499,6 +499,49 @@ func TestSchema(t *testing.T) {
 	}
 }
 
+func TestSync(t *testing.T) {
+	client := getClient()
+	remoteIndex, _ := NewIndex("remote-index-1", nil)
+	err := client.EnsureIndex(remoteIndex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	remoteFrame, _ := remoteIndex.Frame("remote-frame-1", nil)
+	err = client.EnsureFrame(remoteFrame)
+	if err != nil {
+		t.Fatal(err)
+	}
+	schema1 := NewSchema()
+	index11, _ := schema1.Index("diff-index1", nil)
+	index11.Frame("frame1-1", nil)
+	index11.Frame("frame1-2", nil)
+	index12, _ := schema1.Index("diff-index2", nil)
+	index12.Frame("frame2-1", nil)
+	schema1.Index(remoteIndex.Name(), nil)
+
+	err = client.SyncSchema(schema1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	client.DeleteIndex(remoteIndex)
+	client.DeleteIndex(index11)
+	client.DeleteIndex(index12)
+}
+
+func TestSyncFailure(t *testing.T) {
+	server := getMockServer(404, []byte("sorry, not found"), -1)
+	defer server.Close()
+	uri, err := NewURIFromAddress(server.URL)
+	if err != nil {
+		panic(err)
+	}
+	client := NewClientWithURI(uri)
+	err = client.SyncSchema(NewSchema())
+	if err == nil {
+		t.Fatal("should have failed")
+	}
+}
+
 func TestErrorRetrievingSchema(t *testing.T) {
 	server := getMockServer(404, []byte("sorry, not found"), -1)
 	defer server.Close()

orm.go

@@ -68,6 +68,31 @@ func (s *Schema) Index(name string, options *IndexOptions) (*Index, error) {
 	return index, nil
 }
 
+func (s *Schema) diff(other *Schema) *Schema {
+	result := NewSchema()
+	for indexName, index := range s.indexes {
+		if otherIndex, ok := other.indexes[indexName]; !ok {
+			// if the index doesn't exist in the other schema, simply copy it
+			result.indexes[indexName] = index.copy()
+		} else {
+			// the index exists in the other schema; check the frames
+			resultIndex, _ := NewIndex(indexName, index.options)
+			for frameName, frame := range index.frames {
+				if _, ok := otherIndex.frames[frameName]; !ok {
+					// the frame doesn't exist in the other schame, copy it
+					resultIndex.frames[frameName] = frame.copy()
+				}
+			}
+			// check whether we modified result index
+			if len(resultIndex.frames) > 0 {
+				// if so, move it to the result
+				result.indexes[indexName] = resultIndex
+			}
+		}
+	}
+	return result
+}
+
 // PQLQuery is an interface for PQL queries.
 type PQLQuery interface {
 	Index() *Index
@@ -223,6 +248,20 @@ func NewIndex(name string, options *IndexOptions) (*Index, error) {
 	}, nil
 }
 
+func (d *Index) copy() *Index {
+	frames := make(map[string]*Frame)
+	for name, f := range d.frames {
+		frames[name] = f.copy()
+	}
+	index := &Index{
+		name:    d.name,
+		frames:  frames,
+		options: &IndexOptions{},
+	}
+	*index.options = *d.options
+	return index
+}
+
 // Name returns the name of this index.
 func (d *Index) Name() string {
 	return d.name
@@ -379,6 +418,16 @@ type Frame struct {
 	options *FrameOptions
 }
 
+func (f *Frame) copy() *Frame {
+	frame := &Frame{
+		name:    f.name,
+		index:   f.index,
+		options: &FrameOptions{},
+	}
+	*frame.options = *f.options
+	return frame
+}
+
 // Bitmap creates a bitmap query using the row label.
 // Bitmap retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query.
 // It also retrieves any attributes set on that row or column.

orm_test.go

@@ -33,6 +33,7 @@
 package pilosa
 
 import (
+	"reflect"
 	"testing"
 	"time"
 )
@@ -47,6 +48,31 @@ var b2 = sampleFrame.Bitmap(20)
 var b3 = sampleFrame.Bitmap(42)
 var b4 = collabFrame.Bitmap(2)
 
+func TestSchemaDiff(t *testing.T) {
+	schema1 := NewSchema()
+	index11, _ := schema1.Index("diff-index1", nil)
+	index11.Frame("frame1-1", nil)
+	index11.Frame("frame1-2", nil)
+	index12, _ := schema1.Index("diff-index2", nil)
+	index12.Frame("frame2-1", nil)
+
+	schema2 := NewSchema()
+	index21, _ := schema2.Index("diff-index1", nil)
+	index21.Frame("another-frame", nil)
+
+	targetDiff12 := NewSchema()
+	targetIndex1, _ := targetDiff12.Index("diff-index1", nil)
+	targetIndex1.Frame("frame1-1", nil)
+	targetIndex1.Frame("frame1-2", nil)
+	targetIndex2, _ := targetDiff12.Index("diff-index2", nil)
+	targetIndex2.Frame("frame2-1", nil)
+
+	diff12 := schema1.diff(schema2)
+	if !reflect.DeepEqual(targetDiff12, diff12) {
+		t.Fatalf("The diff must be correctly calculated")
+	}
+}
+
 func TestNewIndex(t *testing.T) {
 	index1, err := schema.Index("index-name", nil)
 	if err != nil {
@@ -69,6 +95,28 @@ func TestNewIndexWithInvalidName(t *testing.T) {
 	}
 }
 
+func TestIndexCopy(t *testing.T) {
+	indexOptions := &IndexOptions{
+		ColumnLabel: "columnlabel",
+		TimeQuantum: TimeQuantumMonthDay,
+	}
+	index, err := schema.Index("my-index-4copy", indexOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	options := &FrameOptions{
+		RowLabel: "rowlabel",
+	}
+	_, err = index.Frame("my-frame-4copy", options)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copiedIndex := index.copy()
+	if !reflect.DeepEqual(index, copiedIndex) {
+		t.Fatalf("copied index should be equivalent")
+	}
+}
+
 func TestFrame(t *testing.T) {
 	frame1, err := sampleIndex.Frame("nonexistent-frame", nil)
 	if err != nil {
@@ -83,6 +131,24 @@ func TestFrame(t *testing.T) {
 	}
 }
 
+func TestFrameCopy(t *testing.T) {
+	options := &FrameOptions{
+		RowLabel:       "rowlabel",
+		TimeQuantum:    TimeQuantumMonthDayHour,
+		CacheType:      CacheTypeRanked,
+		CacheSize:      123456,
+		InverseEnabled: true,
+	}
+	frame, err := sampleIndex.Frame("my-frame-4copy", options)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copiedFrame := frame.copy()
+	if !reflect.DeepEqual(frame, copiedFrame) {
+		t.Fatalf("copied frame should be equivalent")
+	}
+}
+
 func TestNewFrameWithInvalidName(t *testing.T) {
 	index, err := NewIndex("foo", nil)
 	if err != nil {