Skip to content
This repository has been archived by the owner on Sep 28, 2022. It is now read-only.

Commit

Permalink
discard slices rather than reslicing them
Browse files Browse the repository at this point in the history
In a previous version of this, I thoughtfully resliced slices
to [:0] rather than allocating. That's a great idea, except that
you don't necessarily ever see a given shard again. If you
don't see that shard again, or if you don't get as many items
in it, all the old record pointers are still sitting there in
reachable parts of the slice's backing store, preventing them
from ever being garbage collected.
  • Loading branch information
seebs committed Oct 25, 2019
1 parent 6683b21 commit 86b9d7d
Showing 1 changed file with 14 additions and 12 deletions.
26 changes: 14 additions & 12 deletions import_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,27 +128,29 @@ func recordImportWorker(id int, client *Client, field *Field, chans importWorker

readRecords:
for recordBatch := range recordChan {
// It's fine to overrun our allowed batch size slightly, and
// we don't want to generate separate batches for part of a
// 16-item batch.
for _, record := range recordBatch {
recordCount++
shard := record.Shard(shardWidth)
if batchForShard[shard] == nil {
batchForShard[shard] = make([]Record, 0, batchSize)
}
batchForShard[shard] = append(batchForShard[shard], record)

if recordCount >= batchSize {
for shard, records := range batchForShard {
if len(records) == 0 {
continue
}
err = importRecords(id, client, field, shardNodes, shard, records, options, statusChan, state)
if err != nil {
break readRecords
}
batchForShard[shard] = batchForShard[shard][:0]
}
if recordCount >= batchSize {
for shard, records := range batchForShard {
if len(records) == 0 {
continue
}
err = importRecords(id, client, field, shardNodes, shard, records, options, statusChan, state)
if err != nil {
break readRecords
}
recordCount = 0
delete(batchForShard, shard)
}
recordCount = 0
}
}

Expand Down

0 comments on commit 86b9d7d

Please sign in to comment.