From 5437432d1ba78e690f5396f3add95b263339b9cd Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 3 Oct 2017 01:57:31 +0300 Subject: [PATCH 1/4] Added TLS support --- README.md | 1 + client.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index d7dcae8..5529e05 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Go client for Pilosa high performance distributed bitmap index. * Added support for excluding bits or attributes from bitmap calls. In order to exclude bits, pass `ExcludeBits: true` in your `QueryOptions`. In order to exclude attributes, pass `ExcludeAttrs: true`. * Added range field operations. * Customizable CSV timestamp format. + * `HTTPS connections are supported. * **Deprecation** Row and column labels are deprecated, and will be removed in a future release of this library. Do not use `ColumnLabel` option for `IndexOptions` and `RowLabel` for `FrameOption` for new code. See: https://github.com/pilosa/pilosa/issues/752 for more info. * **v0.5.0** (2017-08-03): diff --git a/client.go b/client.go index 810e45f..31c580a 100644 --- a/client.go +++ b/client.go @@ -34,6 +34,7 @@ package pilosa import ( "bytes" + "crypto/tls" "encoding/json" "fmt" "io" @@ -616,6 +617,7 @@ func newHTTPClient(options *ClientOptions) *http.Client { Dial: (&net.Dialer{ Timeout: options.ConnectTimeout, }).Dial, + TLSClientConfig: options.TLSConfig, MaxIdleConnsPerHost: options.PoolSizePerRoute, MaxIdleConns: options.TotalPoolSize, } @@ -712,6 +714,7 @@ type ClientOptions struct { ConnectTimeout time.Duration PoolSizePerRoute int TotalPoolSize int + TLSConfig *tls.Config } func (options *ClientOptions) withDefaults() (updated *ClientOptions) { @@ -731,6 +734,9 @@ func (options *ClientOptions) withDefaults() (updated *ClientOptions) { if updated.TotalPoolSize <= 100 { updated.TotalPoolSize = 100 } + if updated.TLSConfig == nil { + updated.TLSConfig = &tls.Config{} + } return } From dff6290ad2ec4ae0006d873abc30c8faca360ff0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 3 Oct 2017 04:15:59 +0300 Subject: [PATCH 2/4] trivial --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5529e05..434ae08 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ GoDoc - + From 0f9603f3e4e10c6d90218e8afa01cc510e756f2d Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 18 Oct 2017 10:15:29 +0300 Subject: [PATCH 3/4] Prefixes node hosts from server with the correct scheme. --- client.go | 52 ++++++++++++++++++++++---------------- client_it_test.go | 18 +++++++++---- gopilosa_pbuf/public.pb.go | 9 ++++--- 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/client.go b/client.go index c70dd9d..4fa72aa 100644 --- a/client.go +++ b/client.go @@ -397,6 +397,7 @@ func (c *Client) importBits(indexName string, frameName string, slice uint64, bi if err != nil { return err } + uri.SetScheme(node.Scheme) err = c.importNode(uri, bitsToImportRequest(indexName, frameName, slice, bits)) if err != nil { return err @@ -417,6 +418,7 @@ func (c *Client) importValues(indexName string, frameName string, slice uint64, if err != nil { return err } + uri.SetScheme(node.Scheme) err = c.importValueNode(uri, valsToImportRequest(indexName, frameName, slice, fieldName, vals)) if err != nil { return err @@ -469,7 +471,7 @@ func (c *Client) ExportFrame(frame *Frame, view string) (BitIterator, error) { if err != nil { return nil, err } - sliceURIs := statusToNodeSlicesForIndex(status, frame.index.Name()) + sliceURIs := c.statusToNodeSlicesForIndex(status, frame.index.Name()) return NewCSVBitIterator(newExportReader(c, sliceURIs, frame, view)), nil } @@ -599,6 +601,31 @@ func (c *Client) doRequest(host *URI, method, path string, headers map[string]st return c.client.Do(req) } +// statusToNodeSlicesForIndex finds the hosts which contains slices for the given index +func (c *Client) statusToNodeSlicesForIndex(status *Status, indexName string) map[uint64]*URI { + // /status endpoint doesn't return the node scheme yet, default to the scheme of the current URI + // TODO: remove the following when /status endpoint returns the scheme for nodes + scheme := c.cluster.hosts[0].Scheme() + result := make(map[uint64]*URI) + for _, node := range status.Nodes { + for _, index := range node.Indexes { + if index.Name != indexName { + continue + } + for _, slice := range index.Slices { + uri, err := NewURIFromAddress(node.Host) + // err will always be nil, but prevent a panic in the odd chance the server returns an invalid URI + if err == nil { + uri.SetScheme(scheme) + result[slice] = uri + } + } + break + } + } + return result +} + func makeRequest(host *URI, method, path string, headers map[string]string, reader io.Reader) (*http.Request, error) { request, err := http.NewRequest(method, host.Normalize()+path, reader) if err != nil { @@ -687,27 +714,6 @@ func valsToImportRequest(indexName string, frameName string, slice uint64, field } } -// statusToNodeSlicesForIndex finds the hosts which contains slices for the given index -func statusToNodeSlicesForIndex(status *Status, indexName string) map[uint64]*URI { - result := make(map[uint64]*URI) - for _, node := range status.Nodes { - for _, index := range node.Indexes { - if index.Name != indexName { - continue - } - for _, slice := range index.Slices { - uri, err := NewURIFromAddress(node.Host) - // err will always be nil, but prevent a panic in the odd chance the server returns an invalid URI - if err == nil { - result[slice] = uri - } - } - break - } - } - return result -} - // ClientOptions control the properties of client connection to the server type ClientOptions struct { SocketTimeout time.Duration @@ -759,6 +765,7 @@ const ( ) type fragmentNode struct { + Scheme string Host string InternalHost string } @@ -774,6 +781,7 @@ type Status struct { // StatusNode contains node information. type StatusNode struct { + Scheme string Host string Indexes []StatusIndex } diff --git a/client_it_test.go b/client_it_test.go index 381dfe1..eddc053 100644 --- a/client_it_test.go +++ b/client_it_test.go @@ -47,6 +47,7 @@ import ( "testing" "time" + "crypto/tls" "github.com/golang/protobuf/proto" pbuf "github.com/pilosa/go-pilosa/gopilosa_pbuf" ) @@ -1332,10 +1333,16 @@ func TestStatusToNodeSlicesForIndex(t *testing.T) { // it needs to access statusToNodeSlicesForIndex which is not // available to client_test.go + uri, err := NewURIFromAddress("https://:10101") + if err != nil { + t.Fatal(err) + } + client := NewClientWithURI(uri) status := &Status{ Nodes: []StatusNode{ { - Host: ":10101", + Scheme: "https", + Host: ":10101", Indexes: []StatusIndex{ { Name: "index1", @@ -1349,12 +1356,12 @@ func TestStatusToNodeSlicesForIndex(t *testing.T) { }, }, } - sliceMap := statusToNodeSlicesForIndex(status, "index2") + sliceMap := client.statusToNodeSlicesForIndex(status, "index2") if len(sliceMap) != 1 { t.Fatalf("slice map should have a single item") } if uri, ok := sliceMap[0]; ok { - target, _ := NewURIFromAddress(":10101") + target, _ := NewURIFromAddress("https://:10101") if !uri.Equals(target) { t.Fatalf("slice map should have the correct URI") } @@ -1364,11 +1371,12 @@ func TestStatusToNodeSlicesForIndex(t *testing.T) { } func getClient() *Client { - uri, err := NewURIFromAddress(":10101") + uri, err := NewURIFromAddress("https://:10101") if err != nil { panic(err) } - return NewClientWithURI(uri) + cluster := NewClusterWithHost(uri) + return NewClientWithCluster(cluster, &ClientOptions{TLSConfig: &tls.Config{InsecureSkipVerify: true}}) } func getMockServer(statusCode int, response []byte, contentLength int) *httptest.Server { diff --git a/gopilosa_pbuf/public.pb.go b/gopilosa_pbuf/public.pb.go index 7f949fb..61a70b6 100644 --- a/gopilosa_pbuf/public.pb.go +++ b/gopilosa_pbuf/public.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. +// Code generated by protoc-gen-go. // source: gopilosa_pbuf/public.proto +// DO NOT EDIT! /* Package gopilosa_pbuf is a generated protocol buffer package. @@ -23,9 +24,9 @@ It has these top-level messages: */ package gopilosa_pbuf -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import "github.com/golang/protobuf/proto" +import "fmt" +import "math" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal From 514f279ac1126da462d75807e6a096c1da0e83c0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 18 Oct 2017 10:16:56 +0300 Subject: [PATCH 4/4] Use http address --- client_it_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client_it_test.go b/client_it_test.go index eddc053..83fa8da 100644 --- a/client_it_test.go +++ b/client_it_test.go @@ -1371,7 +1371,7 @@ func TestStatusToNodeSlicesForIndex(t *testing.T) { } func getClient() *Client { - uri, err := NewURIFromAddress("https://:10101") + uri, err := NewURIFromAddress("http://:10101") if err != nil { panic(err) }