Skip to content
This repository has been archived by the owner on Sep 28, 2022. It is now read-only.

Commit

Permalink
export client KeyTranslation and ImportRoaring methods
Browse files Browse the repository at this point in the history
Now importbatch only uses exported client stuff.
  • Loading branch information
jaffee committed Aug 28, 2019
1 parent c33123a commit 6d88b17
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 14 deletions.
20 changes: 16 additions & 4 deletions client.go
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,7 @@ func (c *Client) translateRecordsRowKeys(rowKeyIDMap *lru.LRU, field *Field, col
}
if len(keys) > 0 {
// translate missing keys
ids, err := c.translateRowKeys(field, keys)
ids, err := c.TranslateRowKeys(field, keys)
if err != nil {
return err
}
Expand Down Expand Up @@ -670,7 +670,7 @@ func (c *Client) translateRecordsColumnKeys(columnKeyIDMap *lru.LRU, index *Inde
}
if len(keys) > 0 {
// translate missing keys
ids, err := c.translateColumnKeys(index, keys)
ids, err := c.TranslateColumnKeys(index, keys)
if err != nil {
return err
}
Expand Down Expand Up @@ -837,6 +837,18 @@ func (c *Client) importData(uri *URI, path string, data []byte) error {
return nil
}

// ImportRoaringBitmap can import pre-made bitmaps for a number of
// different views into the given field/shard. If the view name in the
// map is an empty string, the standard view will be used.
func (c *Client) ImportRoaringBitmap(field *Field, shard uint64, views map[string]*roaring.Bitmap, clear bool) error {
uris, err := c.getURIsForShard(field.index.Name(), shard)
if err != nil {
return errors.Wrap(err, "getting URIs for import")
}
err = c.importRoaringBitmap(uris[0], field, shard, views, &ImportOptions{clear: clear})
return errors.Wrap(err, "importing bitmap")
}

func (c *Client) importRoaringBitmap(uri *URI, field *Field, shard uint64, views viewImports, options *ImportOptions) error {
protoViews := []*pbuf.ImportRoaringRequestView{}
for name, bmp := range views {
Expand Down Expand Up @@ -1156,7 +1168,7 @@ func (c *Client) augmentHeaders(headers map[string]string) map[string]string {
return headers
}

func (c *Client) translateRowKeys(field *Field, keys []string) ([]uint64, error) {
func (c *Client) TranslateRowKeys(field *Field, keys []string) ([]uint64, error) {
req := &pbuf.TranslateKeysRequest{
Index: field.index.name,
Field: field.name,
Expand All @@ -1165,7 +1177,7 @@ func (c *Client) translateRowKeys(field *Field, keys []string) ([]uint64, error)
return c.translateKeys(req, keys)
}

func (c *Client) translateColumnKeys(index *Index, keys []string) ([]uint64, error) {
func (c *Client) TranslateColumnKeys(index *Index, keys []string) ([]uint64, error) {
req := &pbuf.TranslateKeysRequest{
Index: index.name,
Keys: keys,
Expand Down
4 changes: 2 additions & 2 deletions client_it_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2485,7 +2485,7 @@ func TestTranslateRowKeys(t *testing.T) {
if err != nil {
t.Fatal(err)
}
rowIDs, err := client.translateRowKeys(field, []string{"key1", "key2"})
rowIDs, err := client.TranslateRowKeys(field, []string{"key1", "key2"})
if err != nil {
t.Fatal(err)
}
Expand All @@ -2506,7 +2506,7 @@ func TestTranslateColKeys(t *testing.T) {
if err != nil {
t.Fatal(err)
}
colIDs, err := client.translateColumnKeys(keysIndex, []string{"ten", "one-thousand"})
colIDs, err := client.TranslateColumnKeys(keysIndex, []string{"ten", "one-thousand"})
if err != nil {
t.Fatal(err)
}
Expand Down
11 changes: 3 additions & 8 deletions importbatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func (b *Batch) doTranslation() error {
for k := range b.toTranslateID {
keys = append(keys, k)
}
ids, err := b.client.translateColumnKeys(b.index, keys)
ids, err := b.client.TranslateColumnKeys(b.index, keys)
if err != nil {
return errors.Wrap(err, "translating col keys")
}
Expand Down Expand Up @@ -304,7 +304,7 @@ func (b *Batch) doTranslation() error {
}

// translate keys from Pilosa
ids, err := b.client.translateRowKeys(b.headerMap[fieldName], keys)
ids, err := b.client.TranslateRowKeys(b.headerMap[fieldName], keys)
if err != nil {
return errors.Wrap(err, "translating row keys")
}
Expand Down Expand Up @@ -335,16 +335,11 @@ func (b *Batch) doImport() error {

frags := b.makeFragments()
for shard, viewMap := range frags {
uris, err := b.client.getURIsForShard(b.index.Name(), shard)
uri := uris[0]
if err != nil {
return errors.Wrap(err, "getting uris for shard")
}
for fieldView, bitmap := range viewMap {
fieldView := fieldView
bitmap := bitmap
eg.Go(func() error {
err := b.client.importRoaringBitmap(uri, b.index.Field(fieldView.field), shard, map[string]*roaring.Bitmap{"": bitmap}, &ImportOptions{})
err := b.client.ImportRoaringBitmap(b.index.Field(fieldView.field), shard, map[string]*roaring.Bitmap{"": bitmap}, false)
return errors.Wrapf(err, "importing data for %s", fieldView.field)
})
}
Expand Down

0 comments on commit 6d88b17

Please sign in to comment.