From c557aa206fbdf94b26d79528e03ae22edc523f12 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 20 Sep 2019 21:28:22 +0200 Subject: [PATCH 01/12] Add lazy RLE+ decoding License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/internal/bitvector.go | 154 +++++++++++++++++++++ lib/rlepluslazy/internal/bitvector_test.go | 136 ++++++++++++++++++ lib/rlepluslazy/rleplus.go | 110 +++++++++++++++ lib/rlepluslazy/rleplus_test.go | 44 ++++++ 4 files changed, 444 insertions(+) create mode 100644 lib/rlepluslazy/internal/bitvector.go create mode 100644 lib/rlepluslazy/internal/bitvector_test.go create mode 100644 lib/rlepluslazy/rleplus.go create mode 100644 lib/rlepluslazy/rleplus_test.go diff --git a/lib/rlepluslazy/internal/bitvector.go b/lib/rlepluslazy/internal/bitvector.go new file mode 100644 index 00000000000..65bae074e09 --- /dev/null +++ b/lib/rlepluslazy/internal/bitvector.go @@ -0,0 +1,154 @@ +package bitvector + +import ( + "errors" + "log" +) + +var ( + // ErrOutOfRange - the index passed is out of range for the BitVector + ErrOutOfRange = errors.New("index out of range") +) + +// BitNumbering indicates the ordering of bits, either +// least-significant bit in position 0, or most-significant bit +// in position 0. +// +// It it used in 3 ways with BitVector: +// 1. Ordering of bits within the Buf []byte structure +// 2. What order to add bits when using Extend() +// 3. What order to read bits when using Take() +// +// https://en.wikipedia.org/wiki/Bit_numbering +type BitNumbering int + +const ( + // LSB0 - bit ordering starts with the low-order bit + LSB0 BitNumbering = iota + + // MSB0 - bit ordering starts with the high-order bit + MSB0 +) + +// BitVector is used to manipulate ordered collections of bits +type BitVector struct { + Buf []byte + + // BytePacking is the bit ordering within bytes + BytePacking BitNumbering + + // Len is the logical number of bits in the vector. + // The last byte in Buf may have undefined bits if Len is not a multiple of 8 + Len uint +} + +// NewBitVector constructs a new BitVector from a slice of bytes. +// +// The bytePacking parameter is required to know how to interpret the bit ordering within the bytes. +func NewBitVector(buf []byte, bytePacking BitNumbering) *BitVector { + return &BitVector{ + BytePacking: bytePacking, + Buf: buf, + Len: uint(len(buf) * 8), + } +} + +// Push adds a single bit to the BitVector. +// +// Although it takes a byte, only the low-order bit is used, so just use 0 or 1. +func (v *BitVector) Push(val byte) { + if v.Len%8 == 0 { + v.Buf = append(v.Buf, 0) + } + lastIdx := v.Len / 8 + + switch v.BytePacking { + case LSB0: + v.Buf[lastIdx] |= (val & 1) << (v.Len % 8) + default: + v.Buf[lastIdx] |= (val & 1) << (7 - (v.Len % 8)) + } + + v.Len++ +} + +// Get returns a single bit as a byte -- either 0 or 1 +func (v *BitVector) Get(idx uint) (byte, error) { + if idx >= v.Len { + return 0, ErrOutOfRange + } + blockIdx := idx / 8 + + switch v.BytePacking { + case LSB0: + return v.Buf[blockIdx] >> (idx % 8) & 1, nil + default: + return v.Buf[blockIdx] >> (7 - idx%8) & 1, nil + } +} + +// Extend adds up to 8 bits to the receiver +// +// Given a byte b == 0b11010101 +// v.Extend(b, 4, LSB0) would add < 1, 0, 1, 0 > +// v.Extend(b, 4, MSB0) would add < 1, 1, 0, 1 > +// +// Panics if count is out of range +func (v *BitVector) Extend(val byte, count uint, order BitNumbering) { + if count > 8 { + log.Panicf("invalid count") + } + + for i := uint(0); i < count; i++ { + switch order { + case LSB0: + v.Push((val >> i) & 1) + default: + v.Push((val >> (7 - i)) & 1) + } + } +} + +// Take reads up to 8 bits at the given index. +// +// Given a BitVector < 1, 1, 0, 1, 0, 1, 0, 1 > +// v.Take(0, 4, LSB0) would return 0b00001011 +// v.Take(0, 4, MSB0) would return 0b11010000 +// +// Panics if count is out of range +func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) { + if count > 8 { + log.Panicf("invalid count") + } + + for i := uint(0); i < count; i++ { + val, _ := v.Get(index + i) + + switch order { + case LSB0: + out |= val << i + default: + out |= val << (7 - i) + } + } + return +} + +// Iterator returns a function, which when invoked, returns the number +// of bits requested, and increments an internal cursor. +// +// When the end of the BitVector is reached, it returns zeroes indefinitely +// +// Panics if count is out of range +func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { + cursor := uint(0) + return func(count uint) (out byte) { + if count > 8 { + log.Panicf("invalid count") + } + + out = v.Take(cursor, count, order) + cursor += count + return + } +} diff --git a/lib/rlepluslazy/internal/bitvector_test.go b/lib/rlepluslazy/internal/bitvector_test.go new file mode 100644 index 00000000000..3c8f50e3133 --- /dev/null +++ b/lib/rlepluslazy/internal/bitvector_test.go @@ -0,0 +1,136 @@ +package bitvector_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal" +) + +func TestBitVector(t *testing.T) { + t.Run("zero value", func(t *testing.T) { + var v bitvector.BitVector + + assert.Equal(t, bitvector.LSB0, v.BytePacking) + }) + + t.Run("Push", func(t *testing.T) { + // MSB0 bit numbering + v := bitvector.BitVector{BytePacking: bitvector.MSB0} + v.Push(1) + v.Push(0) + v.Push(1) + v.Push(1) + + assert.Equal(t, byte(176), v.Buf[0]) + + // LSB0 bit numbering + v = bitvector.BitVector{BytePacking: bitvector.LSB0} + v.Push(1) + v.Push(0) + v.Push(1) + v.Push(1) + + assert.Equal(t, byte(13), v.Buf[0]) + }) + + t.Run("Get", func(t *testing.T) { + bits := []byte{1, 0, 1, 1, 0, 0, 1, 0} + + for _, numbering := range []bitvector.BitNumbering{bitvector.MSB0, bitvector.LSB0} { + v := bitvector.BitVector{BytePacking: numbering} + + for _, bit := range bits { + v.Push(bit) + } + + for idx, expected := range bits { + actual, _ := v.Get(uint(idx)) + assert.Equal(t, expected, actual) + } + } + }) + + t.Run("Extend", func(t *testing.T) { + val := byte(171) // 0b10101011 + + var v bitvector.BitVector + + // MSB0 bit numbering + v = bitvector.BitVector{} + v.Extend(val, 4, bitvector.MSB0) + assertBitVector(t, []byte{1, 0, 1, 0}, v) + v.Extend(val, 5, bitvector.MSB0) + assertBitVector(t, []byte{1, 0, 1, 0, 1, 0, 1, 0, 1}, v) + + // LSB0 bit numbering + v = bitvector.BitVector{} + v.Extend(val, 4, bitvector.LSB0) + assertBitVector(t, []byte{1, 1, 0, 1}, v) + v.Extend(val, 5, bitvector.LSB0) + assertBitVector(t, []byte{1, 1, 0, 1, 1, 1, 0, 1, 0}, v) + }) + + t.Run("invalid counts to Take/Extend/Iterator cause panics", func(t *testing.T) { + v := bitvector.BitVector{BytePacking: bitvector.LSB0} + + assert.Panics(t, func() { v.Extend(0xff, 9, bitvector.LSB0) }) + + assert.Panics(t, func() { v.Take(0, 9, bitvector.LSB0) }) + + next := v.Iterator(bitvector.LSB0) + assert.Panics(t, func() { next(9) }) + }) + + t.Run("Take", func(t *testing.T) { + var v bitvector.BitVector + + bits := []byte{1, 0, 1, 0, 1, 0, 1, 1} + for _, bit := range bits { + v.Push(bit) + } + + assert.Equal(t, byte(176), v.Take(4, 4, bitvector.MSB0)) + assert.Equal(t, byte(13), v.Take(4, 4, bitvector.LSB0)) + }) + + t.Run("Iterator", func(t *testing.T) { + var buf []byte + + // make a bitvector of 256 sample bits + for i := 0; i < 32; i++ { + buf = append(buf, 128+32) + } + + v := bitvector.NewBitVector(buf, bitvector.LSB0) + + next := v.Iterator(bitvector.LSB0) + + // compare to Get() + for i := uint(0); i < v.Len; i++ { + expected, _ := v.Get(i) + assert.Equal(t, expected, next(1)) + } + + // out of range should return zero + assert.Equal(t, byte(0), next(1)) + assert.Equal(t, byte(0), next(8)) + + // compare to Take() + next = v.Iterator(bitvector.LSB0) + assert.Equal(t, next(5), v.Take(0, 5, bitvector.LSB0)) + assert.Equal(t, next(8), v.Take(5, 8, bitvector.LSB0)) + }) +} + +// Note: When using this helper assertion, expectedBits should *only* be 0s and 1s. +func assertBitVector(t *testing.T, expectedBits []byte, actual bitvector.BitVector) { + assert.Equal(t, uint(len(expectedBits)), actual.Len) + + for idx, bit := range expectedBits { + actualBit, err := actual.Get(uint(idx)) + assert.NoError(t, err) + assert.Equal(t, bit, actualBit) + } +} diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go new file mode 100644 index 00000000000..d24e1ed51b9 --- /dev/null +++ b/lib/rlepluslazy/rleplus.go @@ -0,0 +1,110 @@ +package rlepluslazy + +import ( + "encoding/binary" + "errors" + "fmt" + + bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal" + "golang.org/x/xerrors" +) + +const Version = 0 + +var ( + ErrWrongVersion = errors.New("invalid RLE+ version") + ErrDecode = fmt.Errorf("invalid encoding for RLE+ version %d", Version) +) + +type RLE struct { + vec *bitvector.BitVector +} + +func FromBuf(buf []byte) (*RLE, error) { + rle := &RLE{vec: bitvector.NewBitVector(buf, bitvector.LSB0)} + + if err := rle.check(); err != nil { + return nil, xerrors.Errorf("could not create RLE+ for a buffer: %w", err) + } + return rle, nil +} + +func (rle *RLE) check() error { + ver := rle.vec.Take(0, 2, bitvector.LSB0) + if ver != Version { + return ErrWrongVersion + } + return nil +} + +func (rle *RLE) Iterator() (*iterator, error) { + vit := rle.vec.Iterator(bitvector.LSB0) + vit(2) // Take version + + it := &iterator{next: vit} + if err := it.prep(vit(1)); err != nil { + return nil, err + } + return it, nil +} + +type iterator struct { + next func(uint) byte + + curIdx uint64 + rep uint64 +} + +func (it *iterator) HasNext() bool { + return it.rep != 0 +} + +func (it *iterator) prep(curBit byte) error { + +loop: + for it.rep == 0 { + x := it.next(1) + switch x { + case 1: + it.rep = 1 + case 0: + y := it.next(1) + switch y { + case 1: + it.rep = uint64(it.next(4)) + case 0: + var buf = make([]byte, 0, 10) + for { + b := it.next(8) + buf = append(buf, b) + if b&0x80 == 0 { + break + } + if len(buf) > 10 { + return xerrors.Errorf("run too long: %w", ErrDecode) + } + } + it.rep, _ = binary.Uvarint(buf) + } + + // run with 0 length means end + if it.rep == 0 { + break loop + } + } + + if curBit == 0 { + curBit = 1 + it.curIdx = it.curIdx + it.rep + it.rep = 0 + } + } + return nil +} + +func (it *iterator) Next() (uint64, error) { + it.rep-- + res := it.curIdx + it.curIdx++ + return res, it.prep(0) +} diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go new file mode 100644 index 00000000000..90aaee2e467 --- /dev/null +++ b/lib/rlepluslazy/rleplus_test.go @@ -0,0 +1,44 @@ +package rlepluslazy + +import ( + "testing" + + "github.com/filecoin-project/go-lotus/extern/rleplus" + "github.com/stretchr/testify/assert" +) + +func TestDecode(t *testing.T) { + // Encoding bitvec![LittleEndian; 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + // in the Rust reference implementation gives an encoding of [223, 145, 136, 0] (without version field) + // The bit vector is equivalent to the integer set { 0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 } + + // This is the above reference output with a version header "00" manually added + referenceEncoding := []byte{124, 71, 34, 2} + + expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} + + encoded, _, err := rleplus.Encode(expectedNumbers) + assert.NoError(t, err) + + // Our encoded bytes are the same as the ref bytes + assert.Equal(t, len(referenceEncoding), len(encoded)) + for idx, expected := range referenceEncoding { + assert.Equal(t, expected, encoded[idx]) + } + + rle, err := FromBuf(referenceEncoding) + assert.NoError(t, err) + decoded := make([]uint64, 0, len(expectedNumbers)) + + it, err := rle.Iterator() + assert.NoError(t, err) + for it.HasNext() { + bit, err := it.Next() + assert.NoError(t, err) + decoded = append(decoded, bit) + } + + // Our decoded integers are the same as expected + assert.Equal(t, expectedNumbers, decoded) + +} From e5b3c4757db55cba7d54d6dd0ca403b61541da87 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 21 Sep 2019 19:23:13 +0200 Subject: [PATCH 02/12] More iterative algorithms - Add RunIterator and decoder from RLE - Add BitIterator and BitsFromRuns - Add BitsFromSlice - Add RunsFromBits License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/bits.go | 139 +++++++++++++++++++++++++ lib/rlepluslazy/bits_test.go | 27 +++++ lib/rlepluslazy/interface.go | 24 +++++ lib/rlepluslazy/rleplus.go | 61 +++++++++++ lib/rlepluslazy/rleplus_golden_test.go | 3 + lib/rlepluslazy/rleplus_test.go | 76 +++++++++++++- 6 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 lib/rlepluslazy/bits.go create mode 100644 lib/rlepluslazy/bits_test.go create mode 100644 lib/rlepluslazy/interface.go create mode 100644 lib/rlepluslazy/rleplus_golden_test.go diff --git a/lib/rlepluslazy/bits.go b/lib/rlepluslazy/bits.go new file mode 100644 index 00000000000..b3b0061b891 --- /dev/null +++ b/lib/rlepluslazy/bits.go @@ -0,0 +1,139 @@ +package rlepluslazy + +import ( + "sort" +) + +type it2b struct { + source RunIterator + curIdx uint64 + + run Run +} + +func (it *it2b) HasNext() bool { + return it.run.Valid() +} + +func (it *it2b) Next() (uint64, error) { + it.run.Len-- + res := it.curIdx + it.curIdx++ + return res, it.prep() +} + +func (it *it2b) prep() error { + for !it.run.Valid() && it.source.HasNext() { + var err error + it.run, err = it.source.NextRun() + if err != nil { + return err + } + + if !it.run.Val { + it.curIdx += it.run.Len + it.run.Len = 0 + } + } + return nil +} + +func BitsFromRuns(source RunIterator) (BitIterator, error) { + it := &it2b{source: source} + if err := it.prep(); err != nil { + return nil, err + } + return it, nil +} + +type sliceIt struct { + s []uint64 +} + +func (it sliceIt) HasNext() bool { + return len(it.s) != 0 +} + +func (it *sliceIt) Next() (uint64, error) { + res := it.s[0] + it.s = it.s[1:] + return res, nil +} + +func BitsFromSlice(slice []uint64) BitIterator { + sort.Slice(slice, func(i, j int) bool { return slice[i] < slice[j] }) + return &sliceIt{slice} +} + +type it2r struct { + source BitIterator + + runIdx uint64 + run [2]Run +} + +func (it *it2r) HasNext() bool { + return it.run[0].Valid() +} + +func (it *it2r) NextRun() (Run, error) { + res := it.run[0] + it.runIdx = it.runIdx + res.Len + it.run[0], it.run[1] = it.run[1], Run{} + return res, it.prep() +} + +func (it *it2r) prep() error { + if !it.HasNext() { + return nil + } + if it.run[0].Val == false { + it.run[1].Val = true + it.run[1].Len = 1 + return nil + } + + for it.source.HasNext() && !it.run[1].Valid() { + nB, err := it.source.Next() + if err != nil { + return err + } + + //fmt.Printf("runIdx: %d, run[0].Len: %d, nB: %d\n", it.runIdx, it.run[0].Len, nB) + if it.runIdx+it.run[0].Len == nB { + it.run[0].Len++ + } else { + it.run[1].Len = nB - it.runIdx - it.run[0].Len + it.run[1].Val = false + } + } + return nil +} + +func (it *it2r) init() error { + if it.source.HasNext() { + nB, err := it.source.Next() + if err != nil { + return err + } + it.run[0].Len = nB + it.run[0].Val = false + it.run[1].Len = 1 + it.run[1].Val = true + } + + if !it.run[0].Valid() { + it.run[0], it.run[1] = it.run[1], Run{} + return it.prep() + } + return nil +} + +func RunsFromBits(source BitIterator) (RunIterator, error) { + it := &it2r{source: source} + + if err := it.init(); err != nil { + return nil, err + } + return it, nil +} diff --git a/lib/rlepluslazy/bits_test.go b/lib/rlepluslazy/bits_test.go new file mode 100644 index 00000000000..96eac25fa13 --- /dev/null +++ b/lib/rlepluslazy/bits_test.go @@ -0,0 +1,27 @@ +package rlepluslazy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRunsFromBits(t *testing.T) { + expected := []Run{Run{Val: false, Len: 0x1}, + {Val: true, Len: 0x3}, + {Val: false, Len: 0x2}, + {Val: true, Len: 0x3}, + } + rit, err := RunsFromBits(BitsFromSlice([]uint64{1, 2, 3, 6, 7, 8})) + assert.NoError(t, err) + i := 10 + output := make([]Run, 0, 4) + for rit.HasNext() && i > 0 { + run, err := rit.NextRun() + assert.NoError(t, err) + i-- + output = append(output, run) + } + assert.NotEqual(t, 0, i, "too many iterations") + assert.Equal(t, expected, output) +} diff --git a/lib/rlepluslazy/interface.go b/lib/rlepluslazy/interface.go new file mode 100644 index 00000000000..e9f45910fbb --- /dev/null +++ b/lib/rlepluslazy/interface.go @@ -0,0 +1,24 @@ +package rlepluslazy + +type Run struct { + Val bool + Len uint64 +} + +func (r Run) Valid() bool { + return r.Len != 0 +} + +type RunIterator interface { + NextRun() (Run, error) + HasNext() bool +} + +type RunIterable interface { + RunIterator() (RunIterator, error) +} + +type BitIterator interface { + Next() (uint64, error) + HasNext() bool +} diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go index d24e1ed51b9..0e48f965179 100644 --- a/lib/rlepluslazy/rleplus.go +++ b/lib/rlepluslazy/rleplus.go @@ -37,6 +37,67 @@ func (rle *RLE) check() error { return nil } +func (rle *RLE) RunIterator() (RunIterator, error) { + vit := rle.vec.Iterator(bitvector.LSB0) + vit(2) // Take version + + it := &rleIterator{next: vit} + // next run is previous in relation to prep + // so we invert the value + it.nextRun.Val = vit(1) != 1 + if err := it.prep(); err != nil { + return nil, err + } + return it, nil +} + +type rleIterator struct { + next func(uint) byte + + nextRun Run +} + +func (it *rleIterator) HasNext() bool { + return it.nextRun.Valid() +} + +func (it *rleIterator) NextRun() (Run, error) { + ret := it.nextRun + return ret, it.prep() +} + +func (it *rleIterator) prep() error { + x := it.next(1) + + switch x { + case 1: + it.nextRun.Len = 1 + + case 0: + y := it.next(1) + switch y { + case 1: + it.nextRun.Len = uint64(it.next(4)) + case 0: + var buf = make([]byte, 0, 10) + for { + b := it.next(8) + buf = append(buf, b) + if b&0x80 == 0 { + break + } + if len(buf) > 10 { + return xerrors.Errorf("run too long: %w", ErrDecode) + } + } + it.nextRun.Len, _ = binary.Uvarint(buf) + } + } + + it.nextRun.Val = !it.nextRun.Val + return nil +} + func (rle *RLE) Iterator() (*iterator, error) { vit := rle.vec.Iterator(bitvector.LSB0) vit(2) // Take version diff --git a/lib/rlepluslazy/rleplus_golden_test.go b/lib/rlepluslazy/rleplus_golden_test.go new file mode 100644 index 00000000000..ff4984fa149 --- /dev/null +++ b/lib/rlepluslazy/rleplus_golden_test.go @@ -0,0 +1,3 @@ +package rlepluslazy + +var goldenRLE = []byte{0x40, 0x31, 0x40, 0x84, 0x5c, 0xc0, 0xc6, 0x38, 0xc0, 0xc1, 0x5e, 0x20, 0x92, 0x6, 0x12, 0xbd, 0x9, 0x3a, 0xa3, 0x1, 0x7c, 0x60, 0x97, 0xec, 0x3, 0x8f, 0x4f, 0x1, 0x95, 0x19, 0x26, 0x30, 0x66, 0x80, 0x7, 0x4b, 0x1, 0x7, 0x5f, 0x8, 0xc7, 0x94, 0x28, 0x1f, 0xd0, 0xa9, 0x47, 0x88, 0x7f, 0x80, 0x82, 0x71, 0x82, 0x8, 0x64, 0x82, 0x83, 0x91, 0x46, 0x93, 0x7, 0xb5, 0x40, 0x5f, 0xe4, 0x3b, 0x20, 0x1b, 0x1a, 0x71, 0x44, 0x70, 0x6e, 0x21, 0x81, 0x2e, 0x40, 0x6, 0xfc, 0x7, 0x16, 0xa9, 0x2, 0x16, 0xc8, 0x2, 0x12, 0x34, 0x48, 0xe0, 0x3e, 0xe0, 0x43, 0x2d, 0xa0, 0x52, 0xff, 0xe0, 0xf1, 0xc, 0x1, 0x4, 0xb5, 0x2, 0x44, 0x98, 0x12, 0x7a, 0x26, 0x36, 0xa, 0xa5, 0x2, 0xa9, 0x24, 0x80, 0x2, 0x9, 0x8b, 0x57, 0x1, 0x5, 0xd8, 0x4, 0x11, 0x53, 0x2, 0x95, 0xdc, 0x3, 0x95, 0xc5, 0x0, 0x91, 0xcb, 0x1, 0x36, 0xb8, 0x71, 0xd8, 0xe, 0x50, 0x28, 0xf4, 0x70, 0x18, 0xe, 0xd0, 0x89, 0x64, 0xb0, 0xc0, 0xf, 0x70, 0x18, 0xb3, 0x18, 0xd, 0x48, 0x5, 0x10, 0x84, 0x6, 0x20, 0x41, 0xff, 0x81, 0x8, 0xed, 0x81, 0xc3, 0x6a, 0x82, 0xf, 0xee, 0x80, 0xc2, 0xe4, 0x0, 0x20, 0x34, 0x3f, 0x98, 0x4c, 0x78, 0xd0, 0x1b, 0xf0, 0xc1, 0x34, 0x83, 0x8, 0x78, 0x81, 0x88, 0xb4, 0x80, 0x42, 0xbb, 0x1, 0x10, 0x30, 0x89, 0xbc, 0xf, 0x24, 0x33, 0xc1, 0x82, 0xf4, 0x40, 0xa2, 0x74, 0xc0, 0x4, 0x5f, 0xc1, 0x21, 0x52, 0x40, 0xa1, 0x73, 0xc0, 0x26, 0xfb, 0xc0, 0xe4, 0x4, 0x9, 0xda, 0x9, 0x3c, 0xca, 0x9, 0xa0, 0x20, 0x7a, 0xc0, 0x21, 0x51, 0x0, 0x14, 0xf2, 0xf, 0x34, 0x44, 0x9, 0x58, 0xa1, 0xbe, 0x80, 0xe, 0xe2, 0xd, 0x54, 0x60, 0x7, 0x34, 0x20, 0x44, 0xd2, 0x19, 0x14, 0xbe, 0x1c, 0xfa, 0xd, 0x6c, 0x54, 0x25, 0x88, 0xc0, 0xd5, 0x40, 0x7} diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go index 90aaee2e467..53444a47b0c 100644 --- a/lib/rlepluslazy/rleplus_test.go +++ b/lib/rlepluslazy/rleplus_test.go @@ -1,6 +1,8 @@ package rlepluslazy import ( + "math/rand" + "runtime" "testing" "github.com/filecoin-project/go-lotus/extern/rleplus" @@ -30,7 +32,10 @@ func TestDecode(t *testing.T) { assert.NoError(t, err) decoded := make([]uint64, 0, len(expectedNumbers)) - it, err := rle.Iterator() + rit, err := rle.RunIterator() + assert.NoError(t, err) + + it, err := BitsFromRuns(rit) assert.NoError(t, err) for it.HasNext() { bit, err := it.Next() @@ -40,5 +45,74 @@ func TestDecode(t *testing.T) { // Our decoded integers are the same as expected assert.Equal(t, expectedNumbers, decoded) +} + +func TestGolden(t *testing.T) { + t.SkipNow() + N := 1000 + mod := uint32(1) << 16 + runExProp := float32(0.9) + + bits := make([]uint64, N) + + for i := 0; i < N; i++ { + x := rand.Uint32() % mod + bits[i] = uint64(x) + for rand.Float32() < runExProp && i+1 < N { + i++ + x = (x + 1) % mod + bits[i] = uint64(x) + } + } + + out, _, err := rleplus.Encode(bits) + assert.NoError(t, err) + t.Logf("%#v", out) + _, runs := rleplus.RunLengths(bits) + t.Logf("runs: %v", runs) +} +func BenchmarkIterator(b *testing.B) { + decoded := make([]uint64, 0, 1000) + + for i := 0; i < b.N; i++ { + decoded = decoded[:0] + rle, _ := FromBuf(goldenRLE) + it, _ := rle.Iterator() + for it.HasNext() { + bit, _ := it.Next() + decoded = append(decoded, bit) + } + runtime.KeepAlive(decoded) + } +} + +func BenchmarkRunIterator(b *testing.B) { + runs := make([]Run, 0, 1000) + for i := 0; i < b.N; i++ { + runs = runs[:0] + rle, _ := FromBuf(goldenRLE) + rit, _ := rle.RunIterator() + for rit.HasNext() { + run, _ := rit.NextRun() + runs = append(runs, run) + } + runtime.KeepAlive(runs) + } +} + +func BenchmarkRunsToBits(b *testing.B) { + decoded := make([]uint64, 0, 1000) + + for i := 0; i < b.N; i++ { + decoded = decoded[:0] + rle, _ := FromBuf(goldenRLE) + rit, _ := rle.RunIterator() + it, _ := BitsFromRuns(rit) + for it.HasNext() { + bit, _ := it.Next() + decoded = append(decoded, bit) + } + runtime.KeepAlive(decoded) + } } From 9bf871ee532aa583aad1842e85e727079403111d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 21 Sep 2019 22:40:43 +0200 Subject: [PATCH 03/12] Improve bitvector performance License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/internal/bitvector.go | 88 ++++++++++++++++++++-- lib/rlepluslazy/internal/bitvector_test.go | 21 +++++- lib/rlepluslazy/rleplus_test.go | 16 +--- 3 files changed, 103 insertions(+), 22 deletions(-) diff --git a/lib/rlepluslazy/internal/bitvector.go b/lib/rlepluslazy/internal/bitvector.go index 65bae074e09..fb51eb469f2 100644 --- a/lib/rlepluslazy/internal/bitvector.go +++ b/lib/rlepluslazy/internal/bitvector.go @@ -121,6 +121,22 @@ func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) log.Panicf("invalid count") } + if order == LSB0 && v.BytePacking == LSB0 { + x := index >> 3 + r := index & 7 + var o uint16 + l := uint(len(v.Buf)) + + if x+1 < l { + o = uint16(v.Buf[x]) | uint16(v.Buf[x+1])<<8 + } else if x < l { + o = uint16(v.Buf[x]) + } + + o = o >> r + return byte(o & (0xFF >> (8 - count))) + } + for i := uint(0); i < count; i++ { val, _ := v.Get(index + i) @@ -134,6 +150,18 @@ func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) return } +var masks = [9]byte{ + 0x0, + 0x1, + 0x3, + 0x7, + 0xF, + 0x1F, + 0x3F, + 0x7F, + 0xFF, +} + // Iterator returns a function, which when invoked, returns the number // of bits requested, and increments an internal cursor. // @@ -141,14 +169,60 @@ func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) // // Panics if count is out of range func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { - cursor := uint(0) - return func(count uint) (out byte) { - if count > 8 { - log.Panicf("invalid count") + if order == LSB0 && v.BytePacking == LSB0 { + // Here be dragons + // This is about 10x faster + index := int(0) + bitIdx := uint(0) + + var rest uint64 + for n := 7; n >= 0; n-- { + var o uint64 + if len(v.Buf) > n { + o = uint64(v.Buf[n]) + } + + rest = rest<<8 | o + index++ + } + + return func(bits uint) (out byte) { + if bits > 8 { + log.Panicf("invalid count") + } + res := byte(rest) & masks[bits] + rest = rest >> bits + bitIdx = bitIdx + bits + + if bitIdx > (64 - 8) { + bitIdx = bitIdx & 7 + var add uint64 + + for n := 6; n >= 0; n-- { + var o uint64 + if len(v.Buf) > index+n { + o = uint64(v.Buf[index+n]) + } + + add = add<<8 | o + } + index = index + 7 + rest = rest | add<<(8-bitIdx) + } + + return res } - out = v.Take(cursor, count, order) - cursor += count - return + } else { + cursor := uint(0) + return func(count uint) (out byte) { + if count > 8 { + log.Panicf("invalid count") + } + + out = v.Take(cursor, count, order) + cursor += count + return + } } } diff --git a/lib/rlepluslazy/internal/bitvector_test.go b/lib/rlepluslazy/internal/bitvector_test.go index 3c8f50e3133..c39f7352f2d 100644 --- a/lib/rlepluslazy/internal/bitvector_test.go +++ b/lib/rlepluslazy/internal/bitvector_test.go @@ -1,6 +1,7 @@ package bitvector_test import ( + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -100,7 +101,7 @@ func TestBitVector(t *testing.T) { // make a bitvector of 256 sample bits for i := 0; i < 32; i++ { - buf = append(buf, 128+32) + buf = append(buf, byte(128+i)) } v := bitvector.NewBitVector(buf, bitvector.LSB0) @@ -110,7 +111,7 @@ func TestBitVector(t *testing.T) { // compare to Get() for i := uint(0); i < v.Len; i++ { expected, _ := v.Get(i) - assert.Equal(t, expected, next(1)) + assert.Equal(t, expected, next(1), "at index %d", i) } // out of range should return zero @@ -134,3 +135,19 @@ func assertBitVector(t *testing.T, expectedBits []byte, actual bitvector.BitVect assert.Equal(t, bit, actualBit) } } + +func BenchmarkTake(b *testing.B) { + buf := make([]byte, 0, 128) + for i := 0; i < 128; i++ { + buf = append(buf, byte(128+i)) + } + + v := bitvector.NewBitVector(buf, bitvector.LSB0) + b.ReportAllocs() + b.ResetTimer() + + next := v.Iterator(bitvector.LSB0) + for i := 0; i < b.N; i++ { + runtime.KeepAlive(next(8)) + } +} diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go index 53444a47b0c..e7042011665 100644 --- a/lib/rlepluslazy/rleplus_test.go +++ b/lib/rlepluslazy/rleplus_test.go @@ -73,46 +73,36 @@ func TestGolden(t *testing.T) { } func BenchmarkIterator(b *testing.B) { - decoded := make([]uint64, 0, 1000) - for i := 0; i < b.N; i++ { - decoded = decoded[:0] rle, _ := FromBuf(goldenRLE) it, _ := rle.Iterator() for it.HasNext() { bit, _ := it.Next() - decoded = append(decoded, bit) + runtime.KeepAlive(bit) } - runtime.KeepAlive(decoded) } } func BenchmarkRunIterator(b *testing.B) { - runs := make([]Run, 0, 1000) for i := 0; i < b.N; i++ { - runs = runs[:0] rle, _ := FromBuf(goldenRLE) rit, _ := rle.RunIterator() for rit.HasNext() { run, _ := rit.NextRun() - runs = append(runs, run) + runtime.KeepAlive(run) } - runtime.KeepAlive(runs) } } func BenchmarkRunsToBits(b *testing.B) { - decoded := make([]uint64, 0, 1000) for i := 0; i < b.N; i++ { - decoded = decoded[:0] rle, _ := FromBuf(goldenRLE) rit, _ := rle.RunIterator() it, _ := BitsFromRuns(rit) for it.HasNext() { bit, _ := it.Next() - decoded = append(decoded, bit) + runtime.KeepAlive(bit) } - runtime.KeepAlive(decoded) } } From 3d6071ed3fec1ab9a7b8a03153eb991f53b6d619 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 22 Sep 2019 13:50:43 +0200 Subject: [PATCH 04/12] Improve benchmarks and fix bitvector iterator License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/internal/bitvector.go | 11 ++-- lib/rlepluslazy/internal/bitvector_test.go | 4 +- lib/rlepluslazy/rleplus_golden_test.go | 2 +- lib/rlepluslazy/rleplus_test.go | 63 ++++++++++++++++++---- 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/lib/rlepluslazy/internal/bitvector.go b/lib/rlepluslazy/internal/bitvector.go index fb51eb469f2..e6d1971f871 100644 --- a/lib/rlepluslazy/internal/bitvector.go +++ b/lib/rlepluslazy/internal/bitvector.go @@ -173,7 +173,6 @@ func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { // Here be dragons // This is about 10x faster index := int(0) - bitIdx := uint(0) var rest uint64 for n := 7; n >= 0; n-- { @@ -186,16 +185,17 @@ func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { index++ } + bitCap := uint(64) + return func(bits uint) (out byte) { if bits > 8 { log.Panicf("invalid count") } res := byte(rest) & masks[bits] rest = rest >> bits - bitIdx = bitIdx + bits + bitCap = bitCap - bits - if bitIdx > (64 - 8) { - bitIdx = bitIdx & 7 + if bitCap < 8 { var add uint64 for n := 6; n >= 0; n-- { @@ -207,7 +207,8 @@ func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { add = add<<8 | o } index = index + 7 - rest = rest | add<<(8-bitIdx) + rest = rest | add<<(bitCap) + bitCap = bitCap + 7*8 } return res diff --git a/lib/rlepluslazy/internal/bitvector_test.go b/lib/rlepluslazy/internal/bitvector_test.go index c39f7352f2d..880a3363925 100644 --- a/lib/rlepluslazy/internal/bitvector_test.go +++ b/lib/rlepluslazy/internal/bitvector_test.go @@ -100,8 +100,8 @@ func TestBitVector(t *testing.T) { var buf []byte // make a bitvector of 256 sample bits - for i := 0; i < 32; i++ { - buf = append(buf, byte(128+i)) + for i := 0; i < 1000; i++ { + buf = append(buf, byte(i)) } v := bitvector.NewBitVector(buf, bitvector.LSB0) diff --git a/lib/rlepluslazy/rleplus_golden_test.go b/lib/rlepluslazy/rleplus_golden_test.go index ff4984fa149..091d9826c4b 100644 --- a/lib/rlepluslazy/rleplus_golden_test.go +++ b/lib/rlepluslazy/rleplus_golden_test.go @@ -1,3 +1,3 @@ package rlepluslazy -var goldenRLE = []byte{0x40, 0x31, 0x40, 0x84, 0x5c, 0xc0, 0xc6, 0x38, 0xc0, 0xc1, 0x5e, 0x20, 0x92, 0x6, 0x12, 0xbd, 0x9, 0x3a, 0xa3, 0x1, 0x7c, 0x60, 0x97, 0xec, 0x3, 0x8f, 0x4f, 0x1, 0x95, 0x19, 0x26, 0x30, 0x66, 0x80, 0x7, 0x4b, 0x1, 0x7, 0x5f, 0x8, 0xc7, 0x94, 0x28, 0x1f, 0xd0, 0xa9, 0x47, 0x88, 0x7f, 0x80, 0x82, 0x71, 0x82, 0x8, 0x64, 0x82, 0x83, 0x91, 0x46, 0x93, 0x7, 0xb5, 0x40, 0x5f, 0xe4, 0x3b, 0x20, 0x1b, 0x1a, 0x71, 0x44, 0x70, 0x6e, 0x21, 0x81, 0x2e, 0x40, 0x6, 0xfc, 0x7, 0x16, 0xa9, 0x2, 0x16, 0xc8, 0x2, 0x12, 0x34, 0x48, 0xe0, 0x3e, 0xe0, 0x43, 0x2d, 0xa0, 0x52, 0xff, 0xe0, 0xf1, 0xc, 0x1, 0x4, 0xb5, 0x2, 0x44, 0x98, 0x12, 0x7a, 0x26, 0x36, 0xa, 0xa5, 0x2, 0xa9, 0x24, 0x80, 0x2, 0x9, 0x8b, 0x57, 0x1, 0x5, 0xd8, 0x4, 0x11, 0x53, 0x2, 0x95, 0xdc, 0x3, 0x95, 0xc5, 0x0, 0x91, 0xcb, 0x1, 0x36, 0xb8, 0x71, 0xd8, 0xe, 0x50, 0x28, 0xf4, 0x70, 0x18, 0xe, 0xd0, 0x89, 0x64, 0xb0, 0xc0, 0xf, 0x70, 0x18, 0xb3, 0x18, 0xd, 0x48, 0x5, 0x10, 0x84, 0x6, 0x20, 0x41, 0xff, 0x81, 0x8, 0xed, 0x81, 0xc3, 0x6a, 0x82, 0xf, 0xee, 0x80, 0xc2, 0xe4, 0x0, 0x20, 0x34, 0x3f, 0x98, 0x4c, 0x78, 0xd0, 0x1b, 0xf0, 0xc1, 0x34, 0x83, 0x8, 0x78, 0x81, 0x88, 0xb4, 0x80, 0x42, 0xbb, 0x1, 0x10, 0x30, 0x89, 0xbc, 0xf, 0x24, 0x33, 0xc1, 0x82, 0xf4, 0x40, 0xa2, 0x74, 0xc0, 0x4, 0x5f, 0xc1, 0x21, 0x52, 0x40, 0xa1, 0x73, 0xc0, 0x26, 0xfb, 0xc0, 0xe4, 0x4, 0x9, 0xda, 0x9, 0x3c, 0xca, 0x9, 0xa0, 0x20, 0x7a, 0xc0, 0x21, 0x51, 0x0, 0x14, 0xf2, 0xf, 0x34, 0x44, 0x9, 0x58, 0xa1, 0xbe, 0x80, 0xe, 0xe2, 0xd, 0x54, 0x60, 0x7, 0x34, 0x20, 0x44, 0xd2, 0x19, 0x14, 0xbe, 0x1c, 0xfa, 0xd, 0x6c, 0x54, 0x25, 0x88, 0xc0, 0xd5, 0x40, 0x7} +var goldenRLE = []byte{0x20, 0xfc, 0x40, 0xc2, 0xcc, 0xe5, 0xd8, 0xc1, 0xe1, 0x1e, 0x23, 0xd3, 0x2, 0x2e, 0xcd, 0x3, 0x3e, 0x83, 0x16, 0x26, 0x3a, 0x5c, 0x30, 0x8e, 0x0, 0x5, 0xcc, 0x24, 0xe, 0x96, 0x15, 0x48, 0xa0, 0x2a, 0x40, 0x4, 0x92, 0xe, 0x94, 0xc0, 0x48, 0x91, 0xee, 0x5, 0x28, 0x98, 0x55, 0x90, 0xa0, 0xa4, 0x50, 0x98, 0x14, 0x40, 0x4, 0x59, 0xa, 0x22, 0x0, 0x74, 0xb0, 0x40, 0x66, 0x30, 0xf9, 0x66, 0x90, 0x51, 0xc7, 0x70, 0x74, 0x40, 0x48, 0x7f, 0xf0, 0x80, 0x24, 0x20, 0x85, 0x58, 0x6, 0x7, 0x66, 0x4, 0x87, 0xe5, 0x5, 0x28, 0x0, 0xa4, 0xf0, 0x61, 0x2e, 0x90, 0x8, 0x44, 0x70, 0x38, 0x34, 0xf0, 0x8, 0x4f, 0x70, 0x68, 0xad, 0xd0, 0x90, 0x1e, 0x90, 0x38, 0xc1, 0x85, 0x76, 0x4, 0x15, 0x7c, 0x4, 0x28, 0x28, 0x17, 0x70, 0xe0, 0x15, 0x0, 0x82, 0xfb, 0x11, 0xb, 0x76, 0x9, 0x22, 0xb8, 0x2f, 0x90, 0x20, 0x5f, 0x80, 0x84, 0xc9, 0x10, 0x85, 0x66, 0x9, 0x5, 0xc9, 0x3, 0x2d, 0x19, 0xa4, 0x5a, 0x70, 0x45, 0xa5, 0x40, 0xc2, 0x5, 0x6c, 0x4a, 0xf, 0x64, 0xf4, 0x19, 0xb0, 0x40, 0x28, 0x8b, 0x2, 0x6c, 0x20, 0x3e, 0x90, 0x40, 0x1, 0x19, 0xea, 0x9, 0x20, 0x60, 0x2f, 0x50, 0x60, 0x15, 0x0, 0x4, 0x69, 0x6, 0xb1, 0x8c, 0xa9, 0x85, 0xc6, 0x1f, 0x13, 0x54, 0x3e, 0x58, 0x40, 0x17, 0x60, 0x41, 0x2e, 0x8a, 0x42, 0x3c, 0x8b, 0xb, 0x3f, 0x8, 0x10, 0x5c, 0x32, 0x38, 0xd4, 0x1e, 0x68, 0x18, 0x5b, 0x70, 0xc1, 0x2c, 0x6, 0x17, 0x6c, 0x17, 0xf8, 0x74, 0x36, 0x28, 0x9c, 0xe, 0x20, 0x2, 0xa1, 0x84, 0xf, 0xa2, 0x82, 0xe, 0xbf, 0x82, 0x45, 0x29, 0x84, 0x7, 0x16, 0x25, 0x10, 0x12, 0x38, 0x8c, 0x13, 0x50, 0x41, 0xff, 0x3, 0x13, 0x70, 0x16, 0x80, 0x81, 0x63, 0x85, 0x8f, 0xa0, 0x81, 0xa, 0xe1, 0x80, 0x84, 0xa3, 0x81, 0x89, 0xff, 0x84, 0xc7, 0x3c, 0x87, 0x49, 0xbb, 0x81, 0x4b, 0x21, 0x82, 0x3, 0x2f, 0x83, 0x84, 0x29, 0x83, 0x8c, 0xe0, 0x0, 0x40, 0x8c, 0xda, 0x20, 0x42, 0x6, 0x13, 0x64, 0x7, 0x24, 0x85, 0xc, 0xa6, 0x21, 0x24, 0x6c, 0x7, 0x74, 0x20, 0x13, 0x3c, 0x2, 0x13, 0x54, 0xba, 0x1d, 0x14, 0x9e, 0xb, 0xe0, 0x0, 0x39, 0x82, 0xa, 0x72, 0xb, 0x80, 0x60, 0x53, 0x20, 0xd5, 0xe, 0x32, 0xd5, 0xe, 0x12, 0xb9, 0x1, 0x44, 0x90, 0x27, 0x51, 0x59, 0xe0, 0x12, 0xc3, 0x4, 0xe8, 0x1, 0x78, 0x40, 0xd8, 0xe0, 0x3, 0x7b, 0x60, 0x93, 0x6b, 0x20, 0x72, 0xf8, 0x20, 0xf1, 0xe1, 0xe0, 0x4d, 0x90, 0xdf, 0x2, 0x1f, 0x69, 0x3, 0x36, 0x30, 0x8d, 0x20, 0x82, 0x58, 0xa, 0x8f, 0xec, 0x2, 0x8d, 0xe8, 0x0, 0x11, 0x55, 0x81, 0x58, 0x17, 0xa0, 0x82, 0xa5, 0x81, 0x88, 0xb6, 0x83, 0x88, 0xed, 0x86, 0x88, 0xe0, 0x86, 0x4b, 0x6e, 0x82, 0x45, 0xe1, 0x89, 0xc8, 0x2c, 0x41, 0x78, 0x9, 0x12, 0xbd, 0x91, 0x50, 0x1, 0x42, 0x90, 0x85, 0x70, 0x69, 0xb5, 0x20, 0x83, 0x5e, 0x2, 0x8d, 0xfb, 0x2, 0xf, 0xfb, 0xa, 0x9, 0xcc, 0x4, 0x8b, 0xe4, 0x5, 0x87, 0xc9, 0x2, 0xf, 0xfc, 0x1, 0x5c, 0x8, 0xf, 0xd0, 0x1, 0x4e, 0xd0, 0x60, 0x4d, 0x50, 0xa1, 0x3c, 0xf0, 0x20, 0x6c, 0x88, 0xe1, 0x41, 0x5c, 0x11, 0x1c, 0xc8, 0x25, 0x4c, 0xc0, 0x1d, 0x1c, 0x4e, 0x61, 0x54, 0x2a, 0x5, 0x14, 0xf6, 0x54, 0x3c, 0xb, 0x44, 0xe4, 0x5, 0x64, 0x42, 0x1f, 0x64, 0x32, 0x23, 0xa0, 0x20, 0xf9, 0x1, 0xc, 0x40, 0x27, 0xf8, 0x40, 0x3c, 0xc0, 0x82, 0x72, 0x1, 0xd, 0xd1, 0x43, 0x11, 0x1, 0xd, 0xaa, 0x35, 0xc8, 0xe0, 0x1e, 0x42, 0x41, 0x5b, 0x20, 0x8b, 0x4, 0xe, 0xc9, 0x3, 0xa, 0xea, 0x1, 0x3e, 0x5f, 0x12, 0xab, 0x7, 0x40, 0xf0, 0xad, 0x60, 0x42, 0x6f, 0xa1, 0xd2, 0x39, 0x60, 0x91, 0x64, 0xf1, 0xe4, 0x10, 0x3f, 0x60, 0xc2, 0x5c, 0x94, 0x62, 0x7, 0x1f, 0x67, 0x5, 0x87, 0xcc, 0x7, 0x9f, 0xee, 0xe, 0x13, 0xd8, 0x2, 0x38, 0xa0, 0xb5, 0x20, 0x2, 0x4e, 0x1, 0x85, 0x7c, 0xc, 0x4e, 0xb0, 0xde, 0x90, 0x80, 0x2c, 0x50, 0xd8, 0x77, 0x88, 0xf6, 0x80, 0x4, 0x22, 0x85, 0xce, 0xf2, 0x82, 0xcb, 0xb4, 0x5, 0x14, 0x68, 0xa9, 0x70, 0x77, 0xb8, 0x34, 0x1b, 0xf0, 0x1, 0xec, 0x1, 0x13, 0x58, 0x1a, 0x78, 0x88, 0xb, 0x58, 0x80, 0x27, 0x98, 0x20, 0x4b, 0x50, 0x41, 0x69, 0x84, 0xc9, 0x63, 0x81, 0x8, 0x7b, 0x6, 0x10, 0xd8, 0x1e, 0x38, 0x1c, 0x17, 0x68, 0x0, 0x2e, 0xa8, 0xe4, 0x23, 0x78, 0xdc, 0xa, 0x28, 0x38, 0xb, 0x28, 0xcc, 0x1f, 0x40, 0x1, 0x6f, 0x83, 0x88, 0x75, 0x82, 0xc7, 0x8c, 0x4, 0xec, 0x8, 0x28, 0xcc, 0x7b, 0xe4, 0xb8, 0x0, 0xe, 0x46, 0x35, 0x2c, 0xc2, 0x2d, 0xb8, 0xe0, 0xb5, 0x42, 0xa2, 0x75, 0x40, 0x2, 0xfc, 0xc4, 0x43, 0x98, 0x41, 0xa1, 0x56, 0x40, 0xc4, 0x5a, 0x1, 0x13, 0xda, 0x33, 0x5c, 0xf2, 0x7, 0x44, 0x74, 0x3f, 0x5c, 0x5a, 0x49, 0x24, 0x64, 0x3, 0x2c, 0xd8, 0x23, 0x34, 0x78, 0x19, 0xb8, 0x61, 0x15, 0x42, 0xa3, 0xc6, 0x46, 0x73, 0x80, 0xa, 0x52, 0x17, 0x0, 0xa1, 0x7d, 0x40, 0x26, 0x19, 0xc1, 0x6, 0xf0, 0x40, 0x45, 0x90, 0xc2, 0x81, 0xf6, 0xc0, 0x22, 0xf3, 0x80, 0x15, 0x2, 0x7, 0x2c, 0x9a, 0x3, 0xd8, 0xc0, 0x95, 0x21, 0xbb, 0x4, 0x80, 0x0, 0x99, 0xe0, 0x90, 0x19, 0x21, 0x93, 0xfe, 0x21, 0x43, 0x1a, 0x90, 0x64, 0x2, 0x2e, 0x20, 0x65, 0xc8, 0x79, 0x86, 0xc4, 0x78, 0x4, 0x2a, 0x38, 0x9e, 0x50, 0x82, 0xaf, 0x5, 0x17, 0x50, 0x33, 0xa0, 0x42, 0x21, 0x5, 0x24, 0xc0, 0xf, 0x64, 0xb0, 0x40, 0xe5, 0x3a, 0x42, 0xc1, 0xfd, 0x2, 0x3c, 0x10, 0x9, 0x88, 0x40, 0x5a, 0x40, 0x84, 0x11, 0x41, 0x83, 0x7f, 0x42, 0x82, 0x10, 0x3, 0x8, 0x8a, 0x1b, 0xc8, 0xc0, 0xb2, 0x81, 0x16, 0x6e, 0x68, 0x21, 0x1a, 0x2, 0x11, 0x30, 0x0, 0x42, 0x71, 0xc1, 0xe7, 0x89, 0x18, 0x60, 0x5, 0x3c, 0x70, 0x3c, 0xa2, 0x35, 0x14, 0xe6, 0x25, 0x98, 0x20, 0x97, 0x20, 0xa8, 0x2, 0x26, 0x8e, 0x12, 0x6c, 0x30, 0x2a, 0xa0, 0x22, 0x5e, 0xc2, 0x5, 0x8f, 0x8, 0xa, 0xea, 0x4, 0xa, 0x9b, 0x4, 0x3e, 0xf4, 0x3, 0x2e, 0x92, 0x5, 0x26, 0x65, 0xe0, 0x28, 0xe, 0x38, 0x50, 0xb0, 0x2f, 0x61, 0xe1, 0x6a, 0xa0, 0xf3, 0x2c, 0x12, 0x49, 0x6, 0x26, 0x20, 0x84, 0x30, 0x51, 0x5d, 0x90, 0xb0, 0x57, 0x70, 0x11, 0x3d, 0x70, 0x10, 0x53, 0x1c, 0x98, 0xf8, 0x54, 0x0, 0x2, 0xc3, 0x10, 0x58, 0xb0, 0x4d, 0xb0, 0x20, 0xd, 0x71, 0x40, 0x4d, 0x20, 0x4, 0x79, 0x5, 0x1f, 0x7b, 0x1, 0x89, 0x69, 0x2, 0x28, 0xc8, 0x17, 0x30, 0x81, 0x15, 0xd0, 0xc0, 0x5f, 0x50, 0x19, 0x1e, 0x72, 0x4d, 0x8d, 0xc1, 0x1, 0x91, 0x7e, 0x5, 0x34, 0x30, 0xe, 0x91, 0xc9, 0x74, 0x20, 0x84, 0xd5, 0x13, 0xb, 0x61, 0xd, 0x8f, 0xd8, 0x0, 0x32, 0x88, 0x4e, 0x20, 0x4, 0xd7, 0x84, 0xbc, 0x6, 0x38, 0x6c, 0x1f, 0xc8, 0xe0, 0x12, 0x68, 0xb4, 0x16, 0xa0, 0x41, 0x3a, 0x7, 0x27, 0x90, 0x1a, 0x88, 0x18, 0xa, 0xa5, 0x71, 0x25, 0xf1, 0x9, 0xa8, 0x50, 0x9d, 0xe1, 0x82, 0xbc, 0x81, 0x4, 0xc9, 0x9, 0x7c, 0x10, 0x3e, 0x40, 0x5, 0xe9, 0x2, 0x3a, 0xce, 0xd, 0x5c, 0x70, 0xfa, 0x60, 0x91, 0x41, 0x9, 0xc7, 0x8, 0xa, 0x93, 0x3, 0x7c, 0xa0, 0x3f, 0xe0, 0x53, 0x4b, 0x40, 0x6, 0xea, 0xd, 0xa, 0xcd, 0x18, 0xa, 0x83, 0x5, 0x2a, 0x29, 0xe, 0xc3, 0x1, 0x54, 0xb0, 0x7c, 0x41, 0x4, 0xaf, 0x2, 0x54, 0x40, 0xbe, 0x20, 0x41, 0xa7, 0x21, 0x29, 0xe1, 0x62, 0x6a, 0x81, 0x8, 0xb8, 0x4, 0x64, 0xb0, 0x29, 0xa0, 0xb0, 0x7f, 0xe0, 0x0, 0x3a, 0xc0, 0x5, 0xcb, 0x2, 0xb8, 0xe0, 0x6e, 0xc0, 0x8, 0xda, 0x13, 0x78, 0xe0, 0x1a, 0x0, 0x4, 0xe2, 0x8, 0xa, 0x1b, 0x16, 0x9d, 0x35, 0x88, 0x0, 0xa, 0x61, 0xe3, 0x1d, 0x61, 0xa1, 0x8f, 0xe0, 0xe3, 0x7d, 0x0, 0x8, 0x56, 0x16, 0xb9, 0x5, 0x7c, 0x50, 0x5a, 0xe2, 0xd1, 0xc9, 0x40, 0x8, 0x93, 0xa, 0x36, 0xc4, 0x2, 0xe, 0xf7, 0x15, 0xa, 0xae, 0x1b, 0xa, 0x83, 0xd, 0x36, 0xe8, 0xa, 0x16, 0xde, 0x11, 0x3e, 0x89, 0xb, 0x50, 0x70, 0xd9, 0x10, 0xca, 0x0, 0x38, 0xc0, 0x8f, 0xc1, 0x83, 0x77, 0x86, 0xc, 0x1b, 0xe8, 0x0, 0x9e, 0x78, 0x8, 0xb, 0xa0, 0xc3, 0xb7, 0x84, 0x82, 0x18, 0x4b, 0x54, 0x3b, 0x70, 0x81, 0xbe, 0x41, 0xcc, 0xd, 0x8, 0x61, 0x8e, 0x17, 0x0, 0xd, 0xd8, 0x40, 0x34, 0x20, 0xbd, 0x12, 0xe, 0x83, 0x16, 0x3a, 0x8a, 0x16, 0x48, 0xd0, 0x59, 0xe0, 0x21, 0x4c, 0x60, 0x42, 0xaa, 0xa0, 0x60, 0x69, 0x0, 0x5, 0xb9, 0x2, 0x12, 0x94, 0xf, 0x48, 0x60, 0xfe, 0x90, 0x66, 0x3, 0x4e, 0xf8, 0x4c, 0x50, 0xd8, 0x5f, 0x70, 0x99, 0x1f, 0xd0, 0xf0, 0x55, 0x88, 0x74, 0x81, 0xd, 0x6b, 0x80, 0x84, 0xb9, 0x8d, 0xc7, 0x34, 0x81, 0x4e, 0xa4, 0x8a, 0x82, 0xa1, 0x2, 0x15, 0x40, 0x7, 0x70, 0x81, 0x7f, 0x8d, 0x3, 0x61, 0x2, 0x11, 0x1c, 0x1e, 0x78, 0xb8, 0x1a, 0x80, 0x2, 0xa3, 0x80, 0x2, 0x69, 0x80, 0xc7, 0xf3, 0x1, 0x2c, 0xdc, 0x13, 0x90, 0x81, 0xad, 0x81, 0x88, 0xb8, 0x81, 0x82, 0x78, 0x84, 0x4e, 0xb6, 0x83, 0x45, 0xb3, 0x6, 0x37, 0x90, 0x5e, 0x68, 0x48, 0x82, 0x28, 0x0, 0xc1, 0x1, 0xe5, 0x80, 0x86, 0xf2, 0x81, 0xcc, 0xb9, 0x81, 0x88, 0xeb, 0x80, 0x3, 0xb7, 0x87, 0x8d, 0x69, 0x83, 0x45, 0x74, 0x5, 0x22, 0x84, 0x43, 0x48, 0x48, 0x9f, 0xa4, 0xf3, 0x40, 0xa4, 0x78, 0x80, 0xb, 0xec, 0xb, 0x2c, 0x8, 0x7, 0x74, 0x88, 0x15, 0x34, 0xc2, 0xa2, 0xa1, 0x33, 0xd, 0xc1, 0x8, 0x91, 0x18, 0xf0, 0x60, 0x38, 0x40, 0x7, 0x83, 0x1, 0x8c, 0x10, 0x19, 0x40, 0x4, 0xbb, 0x1a, 0x68, 0x30, 0x2b, 0x60, 0x92, 0xaa, 0x80, 0x8, 0xef, 0x2, 0x54, 0x50, 0x7f, 0x80, 0xa, 0x89, 0x2, 0x32, 0x94, 0x17, 0x1a, 0xe1, 0x4, 0xe, 0xa6, 0x1, 0xa, 0x97, 0xb, 0x16, 0xe4, 0xa, 0x91, 0x3c, 0x90, 0xf9, 0x1e, 0x10, 0xf1, 0x25, 0x70, 0x29, 0x1d, 0xa0, 0x82, 0xf0, 0x4, 0x40, 0xe8, 0x56, 0x40, 0x5, 0x54, 0xf, 0x1b, 0xc6, 0x1, 0x2a, 0x98, 0x35, 0xf0, 0x30, 0xbd, 0x80, 0x7, 0xd9, 0x6, 0x20, 0x90, 0x5e, 0x9, 0xf9, 0xb, 0x1c, 0xac, 0x22, 0x20, 0xc1, 0x96, 0x4f, 0x70, 0x80, 0x4b, 0xae, 0x81, 0x47, 0xb9, 0x8d, 0x8d, 0x3d, 0x82, 0x45, 0x65, 0x80, 0xca, 0xb9, 0x5, 0x18, 0x6c, 0x26, 0xa4, 0x53, 0x40, 0xc7, 0xb2, 0xc2, 0x6, 0xdf, 0x80, 0xa, 0xa0, 0x29, 0x30, 0x23, 0x32, 0x81, 0x8, 0x7c, 0x3c, 0x3e, 0x37, 0xb8, 0x20, 0xd7, 0x80, 0xb, 0xfc, 0x25, 0x24, 0xe8, 0x1d, 0x2c, 0x50, 0xd, 0x3c, 0xe0, 0x88, 0x60, 0x3a, 0x83, 0xc, 0xa6, 0x11, 0xb2, 0x5c, 0xc1, 0x7, 0xf2, 0x1, 0x2a, 0xce, 0x1, 0x12, 0xd7, 0x64, 0xe, 0xb2, 0x7, 0xa9, 0x6e, 0xd0, 0x71, 0x1d, 0x70, 0x80, 0xe5, 0x8, 0x6d, 0x83, 0x84, 0x6a, 0x0, 0x19, 0x78, 0xe, 0x48, 0xc4, 0x7, 0x80, 0x81, 0xae, 0x80, 0x44, 0x3f, 0x1, 0x13, 0x4c, 0x1f, 0x58, 0xe0, 0x4a, 0x48, 0x8c, 0x2e, 0x4, 0x78, 0x81, 0x16, 0x7a, 0x5, 0x90, 0xa0, 0xdd, 0xc3, 0xa7, 0xd8, 0xc2, 0x44, 0xfc, 0x40, 0x2, 0xda, 0x0, 0xc, 0x54, 0x11, 0xe0, 0xc0, 0xba, 0x80, 0x9, 0x7a, 0x9, 0xa0, 0x0, 0x52, 0x40, 0x61, 0x51, 0x44, 0x67, 0x12, 0x3, 0xc, 0x0, 0xf, 0x70, 0xe1, 0x31, 0xc0, 0xe1, 0xb6, 0xc2, 0xa1, 0xb9, 0xc1, 0x46, 0x50, 0x40, 0x1, 0x9b, 0x1, 0x1a, 0xc2, 0x11, 0x72, 0x4c, 0xa1, 0x2, 0x5b, 0x20, 0xb1, 0x28, 0xc0, 0x4, 0xc7, 0x12, 0x40, 0x0, 0xee, 0x0, 0x8, 0xd1, 0x2a, 0x32, 0xd0, 0x11, 0x81, 0x7f, 0xa0, 0x84, 0x77, 0xa, 0x93, 0xc8, 0x0, 0x97, 0xe7, 0x1, 0x9, 0x50, 0x1, 0x5, 0xf5, 0x0, 0x85, 0xee, 0x3, 0x93, 0x59, 0x82, 0x3c, 0x59, 0x14, 0x52, 0x28, 0xa4, 0xb, 0x28, 0x30, 0xa, 0x84, 0x4d, 0x81, 0x7f, 0x43, 0xa7, 0x56, 0x80, 0xb, 0x92, 0xf, 0x3c, 0x98, 0x47, 0xa8, 0x0, 0x35, 0x20, 0xde, 0x3, 0x2a, 0x8b, 0x2, 0x2e, 0x8f, 0x6, 0x58, 0x40, 0x58, 0xe0, 0x33, 0x4d, 0xc1, 0xa, 0xc0, 0x6, 0xc0, 0xc0, 0x4, 0x5, 0xf5, 0x4, 0x16, 0xfa, 0x26, 0x60, 0x60, 0x9c, 0xa0, 0x63, 0x2f, 0x20, 0x13, 0xbe, 0x81, 0xc, 0x97, 0xd, 0x32, 0xd0, 0x1, 0x22, 0x3e, 0x80, 0x20, 0x49, 0xe0, 0x50, 0x18, 0x21, 0x2, 0x19, 0xe0, 0x20, 0xbd, 0x10, 0xc8, 0x4, 0x9b, 0x40, 0xd, 0x24, 0xe0, 0xcd, 0x90, 0xe9, 0x3, 0x82, 0x16, 0x11, 0x28, 0x2a, 0xd0, 0xb5, 0x0, 0x83, 0x73, 0xb, 0x19, 0xd6, 0x7, 0x1b, 0x71, 0x5, 0x26, 0x28, 0x6d, 0x0, 0x82, 0x7f, 0x5, 0x2a, 0xb8, 0x56, 0x80, 0x84, 0x79, 0x4, 0x86, 0xa0, 0x7e, 0x50, 0x20, 0xe3, 0x3, 0x20, 0x36, 0x88, 0xaf, 0x90, 0x91, 0xd5, 0x60, 0x82, 0xe4, 0x1, 0x93, 0xf4, 0x8, 0x95, 0xc0, 0x5, 0x1b, 0x61, 0x6, 0x87, 0xc3, 0x1, 0x47, 0xb5, 0x41, 0x51, 0xc9, 0x26, 0x0, 0x3, 0xc1, 0x0, 0x87, 0x46, 0x5, 0x8f, 0x5a, 0x1, 0x95, 0xef, 0x9, 0x89, 0xca, 0x0, 0x8b, 0xf7, 0x2, 0x19, 0x5b, 0xb, 0x32, 0x8, 0x82, 0x2, 0xec, 0x0, 0x13, 0x23, 0x11, 0x40, 0x4, 0x20, 0xe0, 0x26, 0x51, 0x89, 0x85, 0x40, 0x84, 0xc2, 0x1, 0xf, 0x51, 0x1, 0xd, 0x8, 0x6e, 0xf8, 0x6e, 0x0, 0x82, 0x6a, 0x1, 0xf, 0x72, 0x4, 0x2c, 0x40, 0x4d, 0x50, 0x41, 0x16, 0x8, 0x67, 0x46, 0x4a, 0xd, 0x90, 0xc0, 0xfe, 0xc1, 0x42, 0x30, 0x1, 0x9, 0x50, 0xd, 0x5c, 0x16, 0x9, 0xc0, 0x0, 0xf6, 0x41, 0x27, 0x92, 0xc0, 0x63, 0x7a, 0xc2, 0xa1, 0x9c, 0x40, 0x21, 0x27, 0xec, 0x25, 0x4c, 0x80, 0x1b, 0x20, 0x81, 0x2b, 0xa0, 0x0, 0x18, 0xe0, 0xe1, 0x6a, 0x21, 0x2, 0xa3, 0xd2, 0xaf, 0xe0, 0x3, 0x7a, 0x0, 0x4, 0x99, 0x26, 0x50, 0x70, 0xcf, 0xa0, 0xe1, 0x58, 0xa1, 0xd3, 0x38, 0xa0, 0x30, 0x79, 0x20, 0xd2, 0x3a, 0x20, 0xb3, 0x5e, 0x60, 0xa1, 0xeb, 0x80, 0x5, 0xc8, 0x1, 0x4c, 0xc0, 0x5b, 0xa0, 0x60, 0xc8, 0x41, 0x4, 0x6d, 0xc8, 0x20, 0xab, 0x10, 0xe3, 0x7, 0x1f, 0x4b, 0x7, 0x28, 0x70, 0x17, 0x80, 0x84, 0xf9, 0x6, 0x46, 0x90, 0x44, 0xf0, 0x40, 0xf6, 0x60, 0x5, 0xca, 0x7, 0x1b, 0xcf, 0x2, 0xb, 0x61, 0x12, 0x15} diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go index e7042011665..7486fb0bce6 100644 --- a/lib/rlepluslazy/rleplus_test.go +++ b/lib/rlepluslazy/rleplus_test.go @@ -2,7 +2,6 @@ package rlepluslazy import ( "math/rand" - "runtime" "testing" "github.com/filecoin-project/go-lotus/extern/rleplus" @@ -47,11 +46,11 @@ func TestDecode(t *testing.T) { assert.Equal(t, expectedNumbers, decoded) } -func TestGolden(t *testing.T) { +func TestGoldenGen(t *testing.T) { t.SkipNow() - N := 1000 - mod := uint32(1) << 16 - runExProp := float32(0.9) + N := 10000 + mod := uint32(1) << 20 + runExProp := float32(0.93) bits := make([]uint64, N) @@ -70,39 +69,85 @@ func TestGolden(t *testing.T) { t.Logf("%#v", out) _, runs := rleplus.RunLengths(bits) t.Logf("runs: %v", runs) + t.Logf("len: %d", len(out)) } +func TestGolden(t *testing.T) { + expected, _ := rleplus.Decode(goldenRLE) + res := make([]uint64, 0, len(expected)) + + rle, err := FromBuf(goldenRLE) + assert.NoError(t, err) + rit, err := rle.RunIterator() + assert.NoError(t, err) + it, err := BitsFromRuns(rit) + assert.NoError(t, err) + for it.HasNext() { + bit, err := it.Next() + assert.NoError(t, err) + res = append(res, bit) + } + assert.Equal(t, expected, res) + +} + +var Res uint64 = 0 + func BenchmarkIterator(b *testing.B) { + b.ReportAllocs() + var r uint64 for i := 0; i < b.N; i++ { rle, _ := FromBuf(goldenRLE) it, _ := rle.Iterator() for it.HasNext() { bit, _ := it.Next() - runtime.KeepAlive(bit) + if bit < 1<<63 { + r++ + } } } + Res = Res + r } func BenchmarkRunIterator(b *testing.B) { + b.ReportAllocs() + var r uint64 for i := 0; i < b.N; i++ { rle, _ := FromBuf(goldenRLE) rit, _ := rle.RunIterator() for rit.HasNext() { run, _ := rit.NextRun() - runtime.KeepAlive(run) + if run.Val { + r = r + run.Len + } } } + Res = Res + r } func BenchmarkRunsToBits(b *testing.B) { - + b.ReportAllocs() + var r uint64 for i := 0; i < b.N; i++ { rle, _ := FromBuf(goldenRLE) rit, _ := rle.RunIterator() it, _ := BitsFromRuns(rit) for it.HasNext() { bit, _ := it.Next() - runtime.KeepAlive(bit) + if bit < 1<<63 { + r++ + } } } + Res = Res + r +} + +func BenchmarkOldRLE(b *testing.B) { + b.ReportAllocs() + var r uint64 + for i := 0; i < b.N; i++ { + rle, _ := rleplus.Decode(goldenRLE) + r = r + uint64(len(rle)) + } + Res = Res + r } From d3dfd8a73bd658ca623c667d4b763c3f1063716d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 23 Sep 2019 17:50:41 +0200 Subject: [PATCH 05/12] Add rle encoder License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/rleplus_test.go | 41 ++++++++++++++++++++--- lib/rlepluslazy/rleplus_writer.go | 55 +++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 lib/rlepluslazy/rleplus_writer.go diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go index 7486fb0bce6..c97c14fbbfe 100644 --- a/lib/rlepluslazy/rleplus_test.go +++ b/lib/rlepluslazy/rleplus_test.go @@ -18,14 +18,14 @@ func TestDecode(t *testing.T) { expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} - encoded, _, err := rleplus.Encode(expectedNumbers) + runs, err := RunsFromBits(BitsFromSlice(expectedNumbers)) + assert.NoError(t, err) + encoded, err := EncodeRuns(runs, []byte{}) assert.NoError(t, err) // Our encoded bytes are the same as the ref bytes assert.Equal(t, len(referenceEncoding), len(encoded)) - for idx, expected := range referenceEncoding { - assert.Equal(t, expected, encoded[idx]) - } + assert.Equal(t, referenceEncoding, encoded) rle, err := FromBuf(referenceEncoding) assert.NoError(t, err) @@ -88,7 +88,19 @@ func TestGolden(t *testing.T) { res = append(res, bit) } assert.Equal(t, expected, res) +} + +func TestGoldenLoop(t *testing.T) { + rle, err := FromBuf(goldenRLE) + assert.NoError(t, err) + + rit, err := rle.RunIterator() + assert.NoError(t, err) + + buf, err := EncodeRuns(rit, nil) + assert.NoError(t, err) + assert.Equal(t, goldenRLE, buf) } var Res uint64 = 0 @@ -151,3 +163,24 @@ func BenchmarkOldRLE(b *testing.B) { } Res = Res + r } + +func BenchmarkDecodeEncode(b *testing.B) { + b.ReportAllocs() + var r uint64 + out := make([]byte, 0, len(goldenRLE)) + for i := 0; i < b.N; i++ { + rle, _ := FromBuf(goldenRLE) + rit, _ := rle.RunIterator() + out, _ = EncodeRuns(rit, out) + r = r + uint64(len(out)) + } + + /* + for i := 0; i < b.N; i++ { + rle, _ := rleplus.Decode(goldenRLE) + out, _, _ := rleplus.Encode(rle) + r = r + uint64(len(out)) + } + */ + Res = Res + r +} diff --git a/lib/rlepluslazy/rleplus_writer.go b/lib/rlepluslazy/rleplus_writer.go new file mode 100644 index 00000000000..3c4c4c1943c --- /dev/null +++ b/lib/rlepluslazy/rleplus_writer.go @@ -0,0 +1,55 @@ +package rlepluslazy + +import ( + "encoding/binary" + + bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal" +) + +func EncodeRuns(rit RunIterator, buf []byte) ([]byte, error) { + v := bitvector.NewBitVector(buf[:0], bitvector.LSB0) + v.Extend(0, 2, bitvector.LSB0) // Version + + first := true + varBuf := make([]byte, binary.MaxVarintLen64) + + for rit.HasNext() { + run, err := rit.NextRun() + if err != nil { + return nil, err + } + + if first { + if run.Val { + v.Push(1) + } else { + v.Push(0) + } + first = false + } + + switch { + case run.Len == 1: + v.Push(1) + case run.Len < 16: + v.Push(0) + v.Push(1) + v.Extend(byte(run.Len), 4, bitvector.LSB0) + case run.Len >= 16: + v.Push(0) + v.Push(0) + numBytes := binary.PutUvarint(varBuf, run.Len) + for i := 0; i < numBytes; i++ { + v.Extend(varBuf[i], 8, bitvector.LSB0) + } + } + + } + + if first { + v.Push(0) + } + + return v.Buf, nil + +} From 01c0a6ec9f5448dac546d7a7286af913ac1eb631 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 23 Sep 2019 19:58:58 +0200 Subject: [PATCH 06/12] Optimize and start wrapping it up License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/bitvec.go | 88 +++++++++++++++++ lib/rlepluslazy/rleplus.go | 155 +++--------------------------- lib/rlepluslazy/rleplus_reader.go | 73 ++++++++++++++ lib/rlepluslazy/rleplus_test.go | 38 +++----- lib/rlepluslazy/rleplus_writer.go | 26 +++-- 5 files changed, 199 insertions(+), 181 deletions(-) create mode 100644 lib/rlepluslazy/bitvec.go create mode 100644 lib/rlepluslazy/rleplus_reader.go diff --git a/lib/rlepluslazy/bitvec.go b/lib/rlepluslazy/bitvec.go new file mode 100644 index 00000000000..9bca4a399a1 --- /dev/null +++ b/lib/rlepluslazy/bitvec.go @@ -0,0 +1,88 @@ +package rlepluslazy + +type rbitvec struct { + index int + + bits uint16 + bitCap byte + + vec []byte +} + +func readBitvec(vec []byte) *rbitvec { + bv := &rbitvec{vec: vec} + if len(vec) > 0 { + bv.bits = uint16(bv.vec[0]) + } + return bv +} + +const ( + minCap = 8 + maxCap = 16 +) + +var bitMasks = [9]byte{ + 0x0, + 0x1, + 0x3, + 0x7, + 0xF, + 0x1F, + 0x3F, + 0x7F, + 0xFF, +} + +func (bv *rbitvec) Get(count byte) byte { + res := byte(bv.bits) & bitMasks[count] + bv.bits = bv.bits >> count + bv.bitCap = bv.bitCap - count + + if bv.index < len(bv.vec) { + bv.bits = bv.bits | uint16(bv.vec[bv.index])<> 7 // inc == 1 iff bitcap<8 (+10% perf) + bv.index = bv.index + int(inc) + bv.bitCap = bv.bitCap + inc<<3 + + return res +} + +func writeBitvec(buf []byte) *wbitvec { + return &wbitvec{buf: buf[:0]} +} + +type wbitvec struct { + buf []byte + index int + + bits uint16 + bitCap byte +} + +func (bv *wbitvec) Out() []byte { + if bv.bitCap != 0 { + bv.buf = append(bv.buf, 0)[:bv.index+1] + bv.buf[bv.index] = byte(bv.bits) + } + if bv.bitCap > 8 { + bv.buf = append(bv.buf, byte(bv.bitCap>>8)) + } + return bv.buf +} + +func (bv *wbitvec) Put(val byte, count byte) { + bv.bits = bv.bits | uint16(val)<> 7 // inc == 1 iff bitcap>=8 + bv.index = bv.index + int(inc) + bv.bitCap = bv.bitCap - inc<<3 + bv.bits = bv.bits >> (inc << 3) +} diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go index 0e48f965179..b8f442ce08b 100644 --- a/lib/rlepluslazy/rleplus.go +++ b/lib/rlepluslazy/rleplus.go @@ -1,11 +1,9 @@ package rlepluslazy import ( - "encoding/binary" "errors" "fmt" - bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal" "golang.org/x/xerrors" ) @@ -17,155 +15,34 @@ var ( ) type RLE struct { - vec *bitvector.BitVector -} - -func FromBuf(buf []byte) (*RLE, error) { - rle := &RLE{vec: bitvector.NewBitVector(buf, bitvector.LSB0)} + buf []byte - if err := rle.check(); err != nil { - return nil, xerrors.Errorf("could not create RLE+ for a buffer: %w", err) - } - return rle, nil + changes []change } -func (rle *RLE) check() error { - ver := rle.vec.Take(0, 2, bitvector.LSB0) - if ver != Version { - return ErrWrongVersion - } - return nil +type change struct { + set bool + index uint64 } -func (rle *RLE) RunIterator() (RunIterator, error) { - vit := rle.vec.Iterator(bitvector.LSB0) - vit(2) // Take version - - it := &rleIterator{next: vit} - // next run is previous in relation to prep - // so we invert the value - it.nextRun.Val = vit(1) != 1 - if err := it.prep(); err != nil { - return nil, err - } - return it, nil -} - -type rleIterator struct { - next func(uint) byte - - nextRun Run -} - -func (it *rleIterator) HasNext() bool { - return it.nextRun.Valid() -} - -func (it *rleIterator) NextRun() (Run, error) { - ret := it.nextRun - return ret, it.prep() -} - -func (it *rleIterator) prep() error { - x := it.next(1) - - switch x { - case 1: - it.nextRun.Len = 1 - - case 0: - y := it.next(1) - switch y { - case 1: - it.nextRun.Len = uint64(it.next(4)) - case 0: - var buf = make([]byte, 0, 10) - for { - b := it.next(8) - buf = append(buf, b) - if b&0x80 == 0 { - break - } - if len(buf) > 10 { - return xerrors.Errorf("run too long: %w", ErrDecode) - } - } - it.nextRun.Len, _ = binary.Uvarint(buf) - } - } - - it.nextRun.Val = !it.nextRun.Val - return nil -} - -func (rle *RLE) Iterator() (*iterator, error) { - vit := rle.vec.Iterator(bitvector.LSB0) - vit(2) // Take version +func FromBuf(buf []byte) (*RLE, error) { + rle := &RLE{buf: buf} - it := &iterator{next: vit} - if err := it.prep(vit(1)); err != nil { - return nil, err + if len(buf) > 0 && buf[0]&3 != Version { + return nil, xerrors.Errorf("could not create RLE+ for a buffer: %w", ErrWrongVersion) } - return it, nil -} - -type iterator struct { - next func(uint) byte - curIdx uint64 - rep uint64 + return rle, nil } -func (it *iterator) HasNext() bool { - return it.rep != 0 +func (rle *RLE) RunIterator() (RunIterator, error) { + return DecodeRLE(rle.buf) } -func (it *iterator) prep(curBit byte) error { - -loop: - for it.rep == 0 { - x := it.next(1) - switch x { - case 1: - it.rep = 1 - case 0: - y := it.next(1) - switch y { - case 1: - it.rep = uint64(it.next(4)) - case 0: - var buf = make([]byte, 0, 10) - for { - b := it.next(8) - buf = append(buf, b) - if b&0x80 == 0 { - break - } - if len(buf) > 10 { - return xerrors.Errorf("run too long: %w", ErrDecode) - } - } - it.rep, _ = binary.Uvarint(buf) - } - - // run with 0 length means end - if it.rep == 0 { - break loop - } - } - - if curBit == 0 { - curBit = 1 - it.curIdx = it.curIdx + it.rep - it.rep = 0 - } - } - return nil +func (rle *RLE) Set(index uint64) { + rle.changes = append(rle.changes, change{set: true, index: index}) } -func (it *iterator) Next() (uint64, error) { - it.rep-- - res := it.curIdx - it.curIdx++ - return res, it.prep(0) +func (rle *RLE) Clear(index uint64) { + rle.changes = append(rle.changes, change{set: false, index: index}) } diff --git a/lib/rlepluslazy/rleplus_reader.go b/lib/rlepluslazy/rleplus_reader.go new file mode 100644 index 00000000000..5fff2ccd63b --- /dev/null +++ b/lib/rlepluslazy/rleplus_reader.go @@ -0,0 +1,73 @@ +package rlepluslazy + +import ( + "encoding/binary" + + "golang.org/x/xerrors" +) + +func DecodeRLE(buf []byte) (RunIterator, error) { + bv := readBitvec(buf) + + ver := bv.Get(2) // Read version + if ver != Version { + return nil, ErrWrongVersion + } + + it := &rleIterator{bv: bv} + + // next run is previous in relation to prep + // so we invert the value + it.nextRun.Val = bv.Get(1) != 1 + if err := it.prep(); err != nil { + return nil, err + } + return it, nil +} + +type rleIterator struct { + bv *rbitvec + + nextRun Run +} + +func (it *rleIterator) HasNext() bool { + return it.nextRun.Valid() +} + +func (it *rleIterator) NextRun() (Run, error) { + ret := it.nextRun + return ret, it.prep() +} + +func (it *rleIterator) prep() error { + x := it.bv.Get(1) + + switch x { + case 1: + it.nextRun.Len = 1 + + case 0: + y := it.bv.Get(1) + switch y { + case 1: + it.nextRun.Len = uint64(it.bv.Get(4)) + case 0: + var buf = make([]byte, 0, 10) + for { + b := it.bv.Get(8) + buf = append(buf, b) + if b&0x80 == 0 { + break + } + if len(buf) > 10 { + return xerrors.Errorf("run too long: %w", ErrDecode) + } + } + it.nextRun.Len, _ = binary.Uvarint(buf) + } + } + + it.nextRun.Val = !it.nextRun.Val + return nil +} diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go index c97c14fbbfe..38746c755b3 100644 --- a/lib/rlepluslazy/rleplus_test.go +++ b/lib/rlepluslazy/rleplus_test.go @@ -27,7 +27,7 @@ func TestDecode(t *testing.T) { assert.Equal(t, len(referenceEncoding), len(encoded)) assert.Equal(t, referenceEncoding, encoded) - rle, err := FromBuf(referenceEncoding) + rle, err := FromBuf(encoded) assert.NoError(t, err) decoded := make([]uint64, 0, len(expectedNumbers)) @@ -105,22 +105,6 @@ func TestGoldenLoop(t *testing.T) { var Res uint64 = 0 -func BenchmarkIterator(b *testing.B) { - b.ReportAllocs() - var r uint64 - for i := 0; i < b.N; i++ { - rle, _ := FromBuf(goldenRLE) - it, _ := rle.Iterator() - for it.HasNext() { - bit, _ := it.Next() - if bit < 1<<63 { - r++ - } - } - } - Res = Res + r -} - func BenchmarkRunIterator(b *testing.B) { b.ReportAllocs() var r uint64 @@ -167,20 +151,20 @@ func BenchmarkOldRLE(b *testing.B) { func BenchmarkDecodeEncode(b *testing.B) { b.ReportAllocs() var r uint64 - out := make([]byte, 0, len(goldenRLE)) - for i := 0; i < b.N; i++ { - rle, _ := FromBuf(goldenRLE) - rit, _ := rle.RunIterator() - out, _ = EncodeRuns(rit, out) - r = r + uint64(len(out)) - } - /* + out := make([]byte, 0, len(goldenRLE)) for i := 0; i < b.N; i++ { - rle, _ := rleplus.Decode(goldenRLE) - out, _, _ := rleplus.Encode(rle) + rle, _ := FromBuf(goldenRLE) + rit, _ := rle.RunIterator() + out, _ = EncodeRuns(rit, out) r = r + uint64(len(out)) } */ + + for i := 0; i < b.N; i++ { + rle, _ := rleplus.Decode(goldenRLE) + out, _, _ := rleplus.Encode(rle) + r = r + uint64(len(out)) + } Res = Res + r } diff --git a/lib/rlepluslazy/rleplus_writer.go b/lib/rlepluslazy/rleplus_writer.go index 3c4c4c1943c..306da18d3a5 100644 --- a/lib/rlepluslazy/rleplus_writer.go +++ b/lib/rlepluslazy/rleplus_writer.go @@ -2,13 +2,11 @@ package rlepluslazy import ( "encoding/binary" - - bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal" ) func EncodeRuns(rit RunIterator, buf []byte) ([]byte, error) { - v := bitvector.NewBitVector(buf[:0], bitvector.LSB0) - v.Extend(0, 2, bitvector.LSB0) // Version + bv := writeBitvec(buf) + bv.Put(0, 2) first := true varBuf := make([]byte, binary.MaxVarintLen64) @@ -21,35 +19,33 @@ func EncodeRuns(rit RunIterator, buf []byte) ([]byte, error) { if first { if run.Val { - v.Push(1) + bv.Put(1, 1) } else { - v.Push(0) + bv.Put(0, 1) } first = false } switch { case run.Len == 1: - v.Push(1) + bv.Put(1, 1) case run.Len < 16: - v.Push(0) - v.Push(1) - v.Extend(byte(run.Len), 4, bitvector.LSB0) + bv.Put(2, 2) + bv.Put(byte(run.Len), 4) case run.Len >= 16: - v.Push(0) - v.Push(0) + bv.Put(0, 2) numBytes := binary.PutUvarint(varBuf, run.Len) for i := 0; i < numBytes; i++ { - v.Extend(varBuf[i], 8, bitvector.LSB0) + bv.Put(varBuf[i], 8) } } } if first { - v.Push(0) + bv.Put(0, 1) } - return v.Buf, nil + return bv.Out(), nil } From 794490c4909207802467b2dea049c1f0ce276c70 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 23 Sep 2019 19:59:21 +0200 Subject: [PATCH 07/12] Remove old bitvector License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/internal/bitvector.go | 229 --------------------- lib/rlepluslazy/internal/bitvector_test.go | 153 -------------- 2 files changed, 382 deletions(-) delete mode 100644 lib/rlepluslazy/internal/bitvector.go delete mode 100644 lib/rlepluslazy/internal/bitvector_test.go diff --git a/lib/rlepluslazy/internal/bitvector.go b/lib/rlepluslazy/internal/bitvector.go deleted file mode 100644 index e6d1971f871..00000000000 --- a/lib/rlepluslazy/internal/bitvector.go +++ /dev/null @@ -1,229 +0,0 @@ -package bitvector - -import ( - "errors" - "log" -) - -var ( - // ErrOutOfRange - the index passed is out of range for the BitVector - ErrOutOfRange = errors.New("index out of range") -) - -// BitNumbering indicates the ordering of bits, either -// least-significant bit in position 0, or most-significant bit -// in position 0. -// -// It it used in 3 ways with BitVector: -// 1. Ordering of bits within the Buf []byte structure -// 2. What order to add bits when using Extend() -// 3. What order to read bits when using Take() -// -// https://en.wikipedia.org/wiki/Bit_numbering -type BitNumbering int - -const ( - // LSB0 - bit ordering starts with the low-order bit - LSB0 BitNumbering = iota - - // MSB0 - bit ordering starts with the high-order bit - MSB0 -) - -// BitVector is used to manipulate ordered collections of bits -type BitVector struct { - Buf []byte - - // BytePacking is the bit ordering within bytes - BytePacking BitNumbering - - // Len is the logical number of bits in the vector. - // The last byte in Buf may have undefined bits if Len is not a multiple of 8 - Len uint -} - -// NewBitVector constructs a new BitVector from a slice of bytes. -// -// The bytePacking parameter is required to know how to interpret the bit ordering within the bytes. -func NewBitVector(buf []byte, bytePacking BitNumbering) *BitVector { - return &BitVector{ - BytePacking: bytePacking, - Buf: buf, - Len: uint(len(buf) * 8), - } -} - -// Push adds a single bit to the BitVector. -// -// Although it takes a byte, only the low-order bit is used, so just use 0 or 1. -func (v *BitVector) Push(val byte) { - if v.Len%8 == 0 { - v.Buf = append(v.Buf, 0) - } - lastIdx := v.Len / 8 - - switch v.BytePacking { - case LSB0: - v.Buf[lastIdx] |= (val & 1) << (v.Len % 8) - default: - v.Buf[lastIdx] |= (val & 1) << (7 - (v.Len % 8)) - } - - v.Len++ -} - -// Get returns a single bit as a byte -- either 0 or 1 -func (v *BitVector) Get(idx uint) (byte, error) { - if idx >= v.Len { - return 0, ErrOutOfRange - } - blockIdx := idx / 8 - - switch v.BytePacking { - case LSB0: - return v.Buf[blockIdx] >> (idx % 8) & 1, nil - default: - return v.Buf[blockIdx] >> (7 - idx%8) & 1, nil - } -} - -// Extend adds up to 8 bits to the receiver -// -// Given a byte b == 0b11010101 -// v.Extend(b, 4, LSB0) would add < 1, 0, 1, 0 > -// v.Extend(b, 4, MSB0) would add < 1, 1, 0, 1 > -// -// Panics if count is out of range -func (v *BitVector) Extend(val byte, count uint, order BitNumbering) { - if count > 8 { - log.Panicf("invalid count") - } - - for i := uint(0); i < count; i++ { - switch order { - case LSB0: - v.Push((val >> i) & 1) - default: - v.Push((val >> (7 - i)) & 1) - } - } -} - -// Take reads up to 8 bits at the given index. -// -// Given a BitVector < 1, 1, 0, 1, 0, 1, 0, 1 > -// v.Take(0, 4, LSB0) would return 0b00001011 -// v.Take(0, 4, MSB0) would return 0b11010000 -// -// Panics if count is out of range -func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) { - if count > 8 { - log.Panicf("invalid count") - } - - if order == LSB0 && v.BytePacking == LSB0 { - x := index >> 3 - r := index & 7 - var o uint16 - l := uint(len(v.Buf)) - - if x+1 < l { - o = uint16(v.Buf[x]) | uint16(v.Buf[x+1])<<8 - } else if x < l { - o = uint16(v.Buf[x]) - } - - o = o >> r - return byte(o & (0xFF >> (8 - count))) - } - - for i := uint(0); i < count; i++ { - val, _ := v.Get(index + i) - - switch order { - case LSB0: - out |= val << i - default: - out |= val << (7 - i) - } - } - return -} - -var masks = [9]byte{ - 0x0, - 0x1, - 0x3, - 0x7, - 0xF, - 0x1F, - 0x3F, - 0x7F, - 0xFF, -} - -// Iterator returns a function, which when invoked, returns the number -// of bits requested, and increments an internal cursor. -// -// When the end of the BitVector is reached, it returns zeroes indefinitely -// -// Panics if count is out of range -func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { - if order == LSB0 && v.BytePacking == LSB0 { - // Here be dragons - // This is about 10x faster - index := int(0) - - var rest uint64 - for n := 7; n >= 0; n-- { - var o uint64 - if len(v.Buf) > n { - o = uint64(v.Buf[n]) - } - - rest = rest<<8 | o - index++ - } - - bitCap := uint(64) - - return func(bits uint) (out byte) { - if bits > 8 { - log.Panicf("invalid count") - } - res := byte(rest) & masks[bits] - rest = rest >> bits - bitCap = bitCap - bits - - if bitCap < 8 { - var add uint64 - - for n := 6; n >= 0; n-- { - var o uint64 - if len(v.Buf) > index+n { - o = uint64(v.Buf[index+n]) - } - - add = add<<8 | o - } - index = index + 7 - rest = rest | add<<(bitCap) - bitCap = bitCap + 7*8 - } - - return res - } - - } else { - cursor := uint(0) - return func(count uint) (out byte) { - if count > 8 { - log.Panicf("invalid count") - } - - out = v.Take(cursor, count, order) - cursor += count - return - } - } -} diff --git a/lib/rlepluslazy/internal/bitvector_test.go b/lib/rlepluslazy/internal/bitvector_test.go deleted file mode 100644 index 880a3363925..00000000000 --- a/lib/rlepluslazy/internal/bitvector_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package bitvector_test - -import ( - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - - bitvector "github.com/filecoin-project/go-lotus/lib/rlepluslazy/internal" -) - -func TestBitVector(t *testing.T) { - t.Run("zero value", func(t *testing.T) { - var v bitvector.BitVector - - assert.Equal(t, bitvector.LSB0, v.BytePacking) - }) - - t.Run("Push", func(t *testing.T) { - // MSB0 bit numbering - v := bitvector.BitVector{BytePacking: bitvector.MSB0} - v.Push(1) - v.Push(0) - v.Push(1) - v.Push(1) - - assert.Equal(t, byte(176), v.Buf[0]) - - // LSB0 bit numbering - v = bitvector.BitVector{BytePacking: bitvector.LSB0} - v.Push(1) - v.Push(0) - v.Push(1) - v.Push(1) - - assert.Equal(t, byte(13), v.Buf[0]) - }) - - t.Run("Get", func(t *testing.T) { - bits := []byte{1, 0, 1, 1, 0, 0, 1, 0} - - for _, numbering := range []bitvector.BitNumbering{bitvector.MSB0, bitvector.LSB0} { - v := bitvector.BitVector{BytePacking: numbering} - - for _, bit := range bits { - v.Push(bit) - } - - for idx, expected := range bits { - actual, _ := v.Get(uint(idx)) - assert.Equal(t, expected, actual) - } - } - }) - - t.Run("Extend", func(t *testing.T) { - val := byte(171) // 0b10101011 - - var v bitvector.BitVector - - // MSB0 bit numbering - v = bitvector.BitVector{} - v.Extend(val, 4, bitvector.MSB0) - assertBitVector(t, []byte{1, 0, 1, 0}, v) - v.Extend(val, 5, bitvector.MSB0) - assertBitVector(t, []byte{1, 0, 1, 0, 1, 0, 1, 0, 1}, v) - - // LSB0 bit numbering - v = bitvector.BitVector{} - v.Extend(val, 4, bitvector.LSB0) - assertBitVector(t, []byte{1, 1, 0, 1}, v) - v.Extend(val, 5, bitvector.LSB0) - assertBitVector(t, []byte{1, 1, 0, 1, 1, 1, 0, 1, 0}, v) - }) - - t.Run("invalid counts to Take/Extend/Iterator cause panics", func(t *testing.T) { - v := bitvector.BitVector{BytePacking: bitvector.LSB0} - - assert.Panics(t, func() { v.Extend(0xff, 9, bitvector.LSB0) }) - - assert.Panics(t, func() { v.Take(0, 9, bitvector.LSB0) }) - - next := v.Iterator(bitvector.LSB0) - assert.Panics(t, func() { next(9) }) - }) - - t.Run("Take", func(t *testing.T) { - var v bitvector.BitVector - - bits := []byte{1, 0, 1, 0, 1, 0, 1, 1} - for _, bit := range bits { - v.Push(bit) - } - - assert.Equal(t, byte(176), v.Take(4, 4, bitvector.MSB0)) - assert.Equal(t, byte(13), v.Take(4, 4, bitvector.LSB0)) - }) - - t.Run("Iterator", func(t *testing.T) { - var buf []byte - - // make a bitvector of 256 sample bits - for i := 0; i < 1000; i++ { - buf = append(buf, byte(i)) - } - - v := bitvector.NewBitVector(buf, bitvector.LSB0) - - next := v.Iterator(bitvector.LSB0) - - // compare to Get() - for i := uint(0); i < v.Len; i++ { - expected, _ := v.Get(i) - assert.Equal(t, expected, next(1), "at index %d", i) - } - - // out of range should return zero - assert.Equal(t, byte(0), next(1)) - assert.Equal(t, byte(0), next(8)) - - // compare to Take() - next = v.Iterator(bitvector.LSB0) - assert.Equal(t, next(5), v.Take(0, 5, bitvector.LSB0)) - assert.Equal(t, next(8), v.Take(5, 8, bitvector.LSB0)) - }) -} - -// Note: When using this helper assertion, expectedBits should *only* be 0s and 1s. -func assertBitVector(t *testing.T, expectedBits []byte, actual bitvector.BitVector) { - assert.Equal(t, uint(len(expectedBits)), actual.Len) - - for idx, bit := range expectedBits { - actualBit, err := actual.Get(uint(idx)) - assert.NoError(t, err) - assert.Equal(t, bit, actualBit) - } -} - -func BenchmarkTake(b *testing.B) { - buf := make([]byte, 0, 128) - for i := 0; i < 128; i++ { - buf = append(buf, byte(128+i)) - } - - v := bitvector.NewBitVector(buf, bitvector.LSB0) - b.ReportAllocs() - b.ResetTimer() - - next := v.Iterator(bitvector.LSB0) - for i := 0; i < b.N; i++ { - runtime.KeepAlive(next(8)) - } -} From 353fef3ff6068baa4647f50a6ad567b0b22e087c Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 24 Sep 2019 03:11:56 +0200 Subject: [PATCH 08/12] Improve complex code and comment it License: MIT Signed-off-by: Jakub Sztandera --- docs/rewards.m | 29 +++++++ docs/rewards.m~ | 30 +++++++ docs/rewards1.jpg | Bin 0 -> 27804 bytes docs/rewards2.m | 19 +++++ lib/rlepluslazy/bits.go | 21 +++++ lib/rlepluslazy/bitvec.go | 70 +++++++++++----- lib/rlepluslazy/bitvec.go.bak | 63 +++++++++++++++ lib/rlepluslazy/bitvec_test.go | 21 +++++ lib/rlepluslazy/rleplus.go | 138 +++++++++++++++++++++++++++++++- lib/rlepluslazy/rleplus_test.go | 22 ++--- lib/rlepluslazy/runs.go | 96 ++++++++++++++++++++++ lib/rlepluslazy/runs_test.go | 94 ++++++++++++++++++++++ 12 files changed, 568 insertions(+), 35 deletions(-) create mode 100644 docs/rewards.m create mode 100644 docs/rewards.m~ create mode 100644 docs/rewards1.jpg create mode 100644 docs/rewards2.m create mode 100644 lib/rlepluslazy/bitvec.go.bak create mode 100644 lib/rlepluslazy/bitvec_test.go create mode 100644 lib/rlepluslazy/runs.go create mode 100644 lib/rlepluslazy/runs_test.go diff --git a/docs/rewards.m b/docs/rewards.m new file mode 100644 index 00000000000..28dbe476015 --- /dev/null +++ b/docs/rewards.m @@ -0,0 +1,29 @@ + +HalvingPeriod = vpa(6 * 365 * 24 * 60 * 2); +AdjusmentPeriod = vpa(20160); +Decay = exp(log(vpa(0.5)) / HalvingPeriod * AdjusmentPeriod); +Total = vpa(1400e6); +IV = Total * (1-Decay) / AdjusmentPeriod; + + +syms R(x) +R(x) = IV * (Decay.^floor(x/AdjusmentPeriod)); + +Hmax = HalvingPeriod*5; +heights = linspace(0, Hmax, Hmax/AdjusmentPeriod); +Rewards = R(heights); + +R2 = zeros(size(heights)); + +coffer = Total; +for h = 1:size(heights,2) + k = IV*(coffer/Total); + coffer = coffer - k*AdjusmentPeriod; + R2(h) = k; +end +hYears = heights/2/60/24/365; +plot(hYears, Rewards, 'go', hYears, R2, 'r-') +legend('formula', 'incremental'); + +for6y = Rewards(1:HalvingPeriod/AdjusmentPeriod)*AdjusmentPeriod; +inc6t = R2(1:HalvingPeriod/AdjusmentPeriod)*AdjusmentPeriod; diff --git a/docs/rewards.m~ b/docs/rewards.m~ new file mode 100644 index 00000000000..2f2f57c3776 --- /dev/null +++ b/docs/rewards.m~ @@ -0,0 +1,30 @@ + +HalvingPeriod = vpa(6 * 365 * 24 * 60 * 2); +AdjusmentPeriod = vpa(20160); +Decay = exp(log(vpa(0.5)) / HalvingPeriod * AdjusmentPeriod); +Total = vpa(1400e6); +IV = Total * (1-Decay) / AdjusmentPeriod; + + + +syms R(x) +R(x) = IV * (Decay.^floor(x/AdjusmentPeriod)); + +Hmax = HalvingPeriod*10; +heights = linspace(0, Hmax, HalvingPeriod/AdjusmentPeriod); +Rewards = R(heights); + +plot(heights/2/60/24/365, Rewards, 'r-') + +R2 = zeros(size(heights)); + +coffer = Total; +for h = 1:size(heights,2) + + k = IV*(coffer/Total); + coffer = coffer - k*AdjusmentPeriod; + R2(h) = k; + +end +hYears = heights/2/60/24/365; +plot(hYears, Rewards, 'r-', hYears, R2, 'g-') diff --git a/docs/rewards1.jpg b/docs/rewards1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7fd4bd73e764ecd24348108074d1b28c863853e2 GIT binary patch literal 27804 zcmeHv2Ut`|*7ijZP?8`>4hjg8vj`0>B3VE{a*~`ua;7nWk|j%!jAU9QBT;hBIp>Vz z*mVExhVS1&cV~9KnVtFnXWOUjhI^^Lr|Q%>?>Y6>L5v_~fNRp?QsMv#3IL!W{{aX% zKq4#hemoYIgFs@v|yoybTgN=oSO-gVbkC2j#nu?N)f`W#Qje+JC3oQi&j_6t!9;A5iRMx{bUxdmLp zM?u9$K{No=NC7XS{N)Aw?S*m)740%Q2IiHkSjZcSuK|}(P*E?TphK9V`6L}p# z!@o>$lT!r!x`H0YEn7ma=b_1%v=0jEh!lIb>A3anUR=3)gP4SrjQ%zQ;~gd*UcS5h z_XI>Aiit}|N=Yj{QC3k^Q`aysG%_|ZH8Zz&aCCBZadmt7%E$Nh8$bWB@QBE$=yx%( zDXD4cA2KqtvOg6T6_=Ejl~;VJZ)j|4ZfR}n>mL{#8Xg%Po1L3qSX^3OSzX)N-P=Dn zJUTu({U#R*fcjUlkpKTxunThGBjvh;hK7oU@l7t2OU}rKijQ{rCMP<9hysS5?e$w+ z&oK!fgeDi%U7_Vx+$Peu>%Dq|j%SvB=bLDMk?fZV_TpDb_CJFCU%AErY*ZBFzWW^%lcvYSx{K;DmmB;lons0IOvXXGa z{2R_x{UIxOsR&?ZNEa@Vx~X#q0VD~BAb^+ZkP|9zzLi}0VwViK_bC{TV}t+}+=K_` z5>HWA5Wq2SGy(`+LACy)I?2ge7pCni%IOsMB$hU{qHQhr;Ytk{4(9{h|EH9e)^ z;%ka*W)kjbe2N0Y4YAZ}ENS^PUYhHMC zU8VX@Gc^|n{#PkUj&tsDaMx1P6&`KI;*BNDwzW@BIf52i5rD+%n{FuGV+`DwPnxeNSoxfc^;q>_cE=?FgV{AEoq9){VWQg)nW1a%q2XK)P#x9m`t1a=5A57%vS1NImGt2}y=oT@bqm zeUixUg6I+##RjZ;)e?Uy`k$(s0~H~Y20QaovS+;xPyx(a@87L`vSOx{Ov2muD)%cq zp!WX%b;3V0M1`p)$A)Gb%hXl+r_3jz>++$Kc0i--Guc{ zhxAfBWiBQ^SXAYofAQ6$lE-V;Sh#%|3&l9#p+8YKaxC`;kMg%iBbsU>5o}1Y^^A1csYj1@WGeGr#)j9jVNm z0%_?xts8;R%}xopkE{V4J_gx21()(tpAzwgoSm5=fabO7L{}@JiisjFmamI*JWL87 z$Y=GIG_gpY(0= zVm?@Ivqn;~|jwH8RfAs|h@F=xDX8Tf z*LXmCk952gc=@>TVzVWkF9Zo0H^vr-ioX(6-vd~jUra<-Mc*ZC=4mM z5P*ozZQARKQ39MclTQ1?@*O*zM9VT{PzH-;x%m9mstV~zw}c!qLe&y?Ju$+^DF5lE z7P}1d1#u$)Jq85OC*wb!D{BuIr6NTDAHxyADt#kh)yI{SEb+Ll9u5Dkn2Ee_Nnu%O z|NhBMj0IQr_aq_DZt}xAB@RQbA=enY4AKopB*OI5TdhvXSwOLqkQtigv=W)-c=%w` zz9@24?E){Ef5c9gdnV4$&l};vomL!w!*@#l775NL_QczdBuK>tKegA+Eaj`WW`eA3 ztn!kdd5QVXremy0vw>-~8Z-RaqaT$-Xh_hR>6th)w?C5!<>q?J@s7@_%cZ;2LT*8t zjjdxkT%i6wvbb|=LO%34DgG_{ zp}fW}#mc#HV>ItiUyjjA%9+V$i< zFDOl74K&%we%Yi-FkZi4=Bt@%!?ODzx7j)RW?(kogyCCPOOz(JZIodJH9V;H2Pl1c z(d&(Z>GU{oyz~1VW?ximfsZ8JHI@{aFkJ{yL72Oy>(xnZ+E>mk!GfunGj~t} zsLV~NXhX|);=KEG@@!D94^9rW+)D53DVk&Rd%zp7_%i+p9orp(-Biy-(3n&g?S$vUYfGwQ1;`XMih76PD}fYI9$zv0qB5UnQ!O zK}pN=t0QwP2VCab_o}xkP=`{5rGiPi zz1iCCcl5HgQ;4fAL{d4Eg5p(`#Q=L&+iK%T7?ww0Q2_V78xNxuycA6JaG zZS8spSD4s!bSO4NvDh`h&%z|pPUkAx7*P{mDkO|arM;|+jh7d+iA=TN<>CX{@xcrL zcET5^i*=W$;FmG6pf;bfl25**Pbg+10a{wROF)Jq zZ$ZeFKv09UrR6!+zdRwc7w{gu3C8ibYiIXR!xFQ7qnELJ31e9&M63F>F*}@O^+q$P z=j~~0cMd;EZ~}8Qxy1P62!U~V?cq5)gUpH%*Dr!&>x1Ss*HLFX*VF4YB&5Yrr~ zqNL??b=fMi? zb+F`3DP}KVU1SO36`D$QUaE8+=3AxP%GAoMPr39An8CI6M&XV5VBt&_pguyW%=YLs zy+NtddJQk?%~tkjuN8W6Jnf{t1Z1Y>B0kADcG`9Stl6^c6q;vaT%t~%lU>>-p9MTm zIL=MHY*At}YjvqbH1nk9X=#XUe?FQkJ``4Tr;zHT?jQ+Qu<=IDhhyIdFEMYAhXLe< zO&BxMdpe=Baf6jTd2oH5n~P;aBGl)WszN(>vc%U!ZC@mZrkJ48@o~j_+IO3W%xy-lwiAS34LIXkp?dFB^V=vJ9f^&p|nCH#? zKLm;V(>W&+eLTck)QZpGf6^WpQmEF+4|`-<>&1g2e*yYoyi8Z$$B04o34dJul=3PE z`kGSPM>J}E6r!4+*9g9c1me8~Pf4yx7btZnQ-ow@DBLG3+PvK_9sICJU75})^i_D} zOM6aZT3dd)0tQNZS*sg$DV!(t5@TcwT#{|=ce>O#?vb`qIMF%qdiet{`ns^NY%hPZ z3?!H+(3heMlUfLqqDzpoVE;!iXklFdB1ZtX?!HC<8{}39ASKHC_6}v~SS8r$<-I3+ z=;&Tw!F`p(UxwR1Sj=$GTtTU&u=H34i+8&1059-p4xki6mR;|mZ-|I43Eo#rNDf}0 z3va#;COd)65oD+k&DX3I>t@X?6)lowr#V~5??_DP@|A}j2!;74r%baYr)f^Gqva1# zy?SPT?q0N%Eg~n#J)W+}OOrtvTc#jJWaRVR%K7I3+;=hNZ~V6;-x=5f5Hfv9j+dd2I>SF0);drXPNmiHShtDqr?u{((LN#)^+$Q$#furT4 zR@#}es7;s@+{9Ml#9t-0k?T=4+wJ*3T_OMZxZyXjz<<-DUy#0_O=$@XPLA}&6v9kT z@t)Y}R2j3Nv`T5GGvCCe+-Z`zd5(8}pp*4eeYOZV_WRIcBvV#kAoUy1u z1q@X8nz5G5JVs%1M;(ALopfNGmIyqd>Xsk$!svRMIJjdTzV~|as|c*$4DxDYCB^@g zdj*N;esLUm>36>Ow?rKc+cvN_Q`ATdZdD5R*BHO(ypS1lD`1)c_iV z3BY5Zg4^j|51pzr(_i>`j}J2<;kpN0c|9;WB5Oq)$E6Euda6==OCPh z$YaYELoSySA09jCMJx>A%V1PirX7SV^c+QZHpem#9J(=sA6CPAdpEa87Tb`){S*aR z57aAuQ1VW*N!FHP7THH-+Co+29$#(`V~RiUXJ`$-Mf_svGsM7|E zDSGSTcF3RG5q+rdPFmTnU}I?DN0==B!1sZi^Y!xJ$}}YC1bwyT zdG6%ve2<@$AJzLF2=~KB5YcQWE*Q&{?1kx=-BcWwt8fqNW2wrfw2}=I1|f_5AFuqY z0=;5@3j*-lKj;J9mrlmd+0x9w||CJ&g7e%yh>K&cRM{@ZAt)}R%<-XMzg9w>Y0WY0CWJNdKg>;gV zRzolk+`a<>O}*wxr9Z41KdI6?l45n|bIVn{k}Rc~nyJ^FCtTi*ZhmV|;u`l>7;13U zll}bQsz6>Nod|Qb%ma>hK5vrAa9c)Qf^-HKiEGc0b0Lkaco)a|cF%yZ4CJV}*_fiZaNZAfwL1UGd?a|e@BQ|k%3i9w6z(*KtN-glToSLubjOOR1YV(Ab0dsq zSGPh+I|LxF=XIp?Y=uMe2|kC`yJn{upiu}Q0QQdIj(MAxP1)-vP$V;=!;uk=yIJy8rdD><3OU*#|^9@Fn0Tn~HgI5(Y<{}F>=2Twd zTxX6!OVF82Goe35C48-0gjJOLWx4wX0x%jstG}szW*hH09e1w@W!WDXxafcK1er&8 zv227dNpBfGt-q`;g`MU^^MaZCACZkIl4bA@xV$o_S;JnS_}bvWet(o#fx=9y$pnT7Sb8GF3- z8;|Q*o`n*-SeKsC+&K_!ffx6yhPsF~U6MKXvVE%cqw4bH)BKO!UOHL79d zBeUy1-K}}n4X*pJP62;lKDnRwj>BEQ{S&KQaHSOPHXSLs9;~PU1DS_vG5o(l&w)3EWekX2H0?8v9>y%ncNK=k?{|h_@EH z?F8YdtH}P>|B&$ zwZ;c7Z&SYF(Mfzjp-X$Lw1@yojhu-QfOXrBO`$|auNcw(a{uq09r>62KOW#Yi)+PC z8e-4A6i8GeDQRQBB;LH+sGH)=0iq-qO9I(2ceG~e-U+S6UeK0l9Rpg|Yf(p3&VQ9< z_-lan(>UZunJ&JS3$dL4(iHP&GuEa1J1w8p$adn!U5l(7a$tF%+x5Yf_f%kzFID+v z)NeE-;y&%Ts1vvAV@%q%m7eTFi3RHS`jn$~rjg8`@PVB|fBa|4>7V*^XFVaC!)EL} zPGsUqyh3bP$@N^b&8fIJI#8iw&z$SDqa6*dsW-X5v{Z5+0O1!^S3f@^3CVq|k%-yY|7g`eYS@41^Y>G6F$8t^#b!q3ryP-?$%|=>D;L1_wtFGPwx-}j1b`)*S5?P1dD*RynHjZ(M#w=jhKlDaC)<- z9bZrB_U}XEYNcj&Z198<5*OeWqRSlIQP;0;EH7fLMVG;Ck30>QoQsf|YR>)SpR-)Q zV)7PK=d%Q02(|*Rokyv2*>zQ+c7=$ZkZrB^)#-Z`h{@-7 z4`;2+*y2_ZE~iC>X{VJUu}wwS$CL}io&V6!s23dEuOx+t8;P+rFg*{Gs=btQrd>K( z-GBI|GSH-Rif?|s&-?awm?hoF5j&Su&G4EXX0Gl9n#x}?Y2N|9)LgUnY%_a}lDizp zJlPFOypv7U%*=RNMK*%bP{nw2kuT<}D=rc2d?9(hV+w(BB_Eqci8EpM&vD#uC%RLm z>fsD%u>{1sQO%XvP_G!4j?d|%b)ZsjNn*4?r#x|0r| z@wHuQESmKLqpNWFrJ@62cML_fNAiW4M@71MRjrWmUM*;+MUD_UGH&?r$Hm|mP1~-jny5`5#Y;P7LH8&L( zF5PeG_P|TZJH4nyzpI1%JH&r-*Qr2fOvZ|%0~zuo84`K2Vz(nA6TU<|2@e=eV9!;T zNwMIvrhD}?jDu43Soh%EGiLyFH-}PFM)mDWTjExR@(l>}PA&oO6i7V^j>B+*y z+iC8S7;uE0=lE@x6yFZWB1tnb9lkn)04PjSJtg*!pzl6RX$IIv*CGJ6ki|k_V_bEw zn|mJ263gi4!|UiLy5TClR8?ZUl!jssPups*N!ZFkGlky^{t+8||M88yp9BOK;2Nbx zYo&IIRGRrMHDb;^IyK~+gs*|WR2diYZgQWgN=03!uGUSUh4&x;Dv5a=C!<%Vjg-Rl z$b=Vr%(5}RszxG+Xpy{lsiL`0MwGlxU~F0~fZ=H;!Iod)5t2fe2X|9d*STY4FYJ?> z-_i8Y=Peaq&Cbo!z2>2+0QssAg^#{#Y&XQZgz#*&PQ(jSHa9QV4x9~ArupQ z>lJffWC+i3-ZF>vPnfwBlqwT%iamt^DzWZ6aF=zJF>{Lg`XodK>BUyNKm73NemfzDw^g;X?Y9>IuG4ixasZoLqrNGqsIhXr~7fa%$oJE5ATokl!L1kUy z~^x&Dnk;>VQq5|6AU#IbX^Fk5J-M5u}-;P{^*JQHzo;qHge?bxbiXJ5Fw zpjuB@(iu?h4aWk}4+e||So5PiKUipHS-p>VD%><%FisPAwp8Fjls{VIQ-pOBH+AX! z#gtI9Tp9j?c+GClhYt5;jG+o4uzY}Q2}eWM;)1JT=uY{hA4@ZUoAD7)f&7xQ3V)B2 z{`QU?%ed?_8!0NjrW|pEk>^VMX5Y_2NbS?5Hgr{rm|NZOSNG06Ek>BRwfks?c$QxI zTxajR9(p8yOqB@F-m?c051CDY3C6oy_Uu{6^Kf05+l0U|jsXX&qieZ_F5m{;F{in{ zIWe2T05~n|C_lSW?g|?ZtGpfegNN~kkeqPP%MD24P?f#-AnoA(Lir<%vrp_t znl-apCvt*g%_9R|yW^dZ*n7LVCIW|C&g(ahEDGWKP7trGi5{f#jR_^C{A)A!Z}UHY z)L%B4Z-LbZNk5QkmzJStY?E2@oPjv%PR{;g^nbyX`p;}#{oLZa$4<6##nW)OR~-9I zSVXI{MWpv1NfG_Qsi40j-HY!}oNw%CYgS%Qa*5ZAI3l7ajFnaQOQ$y} zSP-V|$>*_pSZOZ3w|Lh~=`8IuWO>6PYum&xYKiV+d4$6_>Rnyg*VFyG`}sMF{j&ob zhQ;r&IZiZ587JRBX$KwFt1uqh(z%dM%!-TRcGV28t}!oh5}F$o4{GRQ!I5BD+@sN) zi0gDGlC>JiAx9paJ8bS<*&iQW$+xekx*eeUoF-;K+oy$Qhw*?4p1O1D;JE*U`Aa2{ zw2kP>vtxf3fAjRa84G9NZxN2cvmdRd-|%<8QGx=w`|%xpuw?xg2A!9Fgi1aYZ<6xn z;4wdwdb{Px=PgpZJP51ubNFfn?=~VOLVD?Zgp^G*zlaknBy)?EUyy+d(VDe`V$Deyy+Y zz8S+;Qfr8@BLPpG4?K+u%^6?6q3?oAVoPVZ`x+*MV|ZLxpOZDt+gZsp%*3&Z0EAR2 zIVXarqtjjCahX%5cUNyoj$oLYY=M!;85u5VnJg!(^JPU$_LFQcn)?%bD_~?N8hWZ!=8C+f-3Y3xlN-((KEkY@;vi)PzLMj$I|f4 zOM_@}9p*CDO?LJ}-U4gC1`{t;O3cc5Dv}zP1gWf+8^K9EMKR6IpnjFyR0UY_m$Ii&Uf@YdBp>aA7hH3dbWDlA-0u^=WqpO3hoL8U9q#6EQr1?j zsu{VCaeV3oiPg7vk-I!uk9;boRsR&BH~Q zyg4?r!K)Zs3(YH@HPL$`-#iCV(qBABPlgWC;%IX>T3ddFfn7W7nit6rTOtTJn>G}hn-dXSPufY(dM!51>e6X3a3h`ELHDecVM2Z zQ)Q2ufrWbs#bul{dQ3|5?zScL#Z`jys1uDnHZ>f#i9A!HLNz3 z!=3cBsMW2SPA7NHeFZVc_s@UhZKXeYE9n=TN>N9%%NsbCJ7id_2Gt<(WU>CTcbJ8I z7_->anN!WexhiaWRrnW?YXwLoTPkVprIiD5~qFGRKjb zm5o1m7H*0FY)?jW3P1E6Sjg&3w40xdJ!_fZO0-qYX{4#sQ$c3DFX6{s;T1Ry(055qM~SI2DSBsQjKL$mT~x#&BLM2L-;y; zy>P9_boDRUHpAaC+xF*%4L<<5sLbj;R=n?;W3lZSzeuNsZg`5hMitDiZ{lu5*5b9$ z3O`AlnO=@PWHS8N43eej%?&M8)_ZHw(X<$`BfQ(EeRi5)%+2a!yo^J4)VHRisO{;j z`N0+X+QW)Bj$E^s?hVFiDOkX%EnyzXU3{&VIku)UXQ3_%wVm{Qv#DiVGb6jUr{=MC z#puoUDtsQ2*i_E!elGE#f(t!T`kA&Wc6IV_84P@Nch!KC{|b}x;QYDe>8X%BUobQN z%h+_rRahSKY#zax?yT2gtJYyQb7U{b%FHV}+UTYeHA}W#5S(ZrhdI;eW<=l%ToIyu z5)WCHhpt(l6Sk;{Wbn!R-K{I2$`v=h9q9D_!RUMkyng1MKP~M4Q7G{D-+d~E;EkHE zS15CK2|S9wJHN)P;@_3Ws2oB!*c)livo7)_rNSi4Lil=h-)+}OE6-oR8yWmU*Y z>xWe;mH{5Cw{3Mw{P-=-;*d$9>Ub~S^}y^kWp5+eZwJclPKRIQeob*IRDPkdPKp3j zreE9C??Tb5l~e+&@^9#})i4bACQ#9`d^Ne+|Q=F z*5%fCvst*g%2G8@*R&!yfUTcgyyi<7MKS_#{OnaDJ<1A?LUCco%70F}|m}phM0ZTWP_Z{#oinO(V{5oN9$Wnx+$By}s&NZgtWj~48Xk>I-Y2}YDY#g5Dz zhC|I+iAPw+H_};Ql_bN9EdgE_XQM|+xD|ib!{6u~T?X5}p|_K)mE2K<6ulhBzP98@ zA^M{-#yt*6yNvVx0qVU*rzfhyv{z)V8P?VI>(t&Syx2sf#t zP}06b`i@%Nn)QRcP3q=om9CH4z7Y^R8{~O*^slGGsS{)4QxEjZedv+-BP{G;+8hsX zNd?ZK6ycra)_2yVhe8tTI)u9&DmS!_&2+aRyoDf*7%v7qu$k0!7oXw@)EDw9Zq0!V z^2TBEA*27SqG)q8L)$2lyJ)T&Fl}0q5WqI9s*o6IEEs_RD&`RYR6|2G=xb5u$4SGC z`{4=T8TNCq^pLOJJlms(marP-nhC4dohF^FilCv5M6QFE!-F~L69?wZiTxhhAEp`A z6F2%!CB2mP&8PoTZtO265o#WME7EjNDY9F`mnrxbXdo^2n>7sj@^8-ul1QI1F(ny< zUf}tZ@M+$s2cqtYTzudt#!HDmt=N%^haui;ibvkI$7N33X7~*C9!j@`aD*HKc7R*ImyPu!p zzYO(B)z(O}Id{HSKq5dAXP)YHKi3w&|AB z>h6~E_)zSnH7p^R?jhpq&^RGI!q0WzJ|t6!IlEvm*O7kG;#sgeRGj+xQkj+f?8FI| zCdfXt=jCGSF*Ur(URA&=C#{P%{Fz>hdQ<-~$NC9bqrpy@EOXO(H*Clki`xIos;YH= z|3U_NBLWz2V<_&`k;yooR_uSTGP^JW_MNd-LI9|gU;^fkA9n_Q+Mr{!F7_~UXXw(s zbLPI}xY;!i`N$IlkkFitJropOB1eqeNH~?!VgD+S%~VxL)7)`ew11N0wf%Y)@4pLxLQ&K`4UK z*PPyaAg8+ewS0sBforn=95+R8+8ei=e34w{ZVWS3*$N(aCkBQ#YrGlzQ`H!jRD7{L zz2#HzE@*X8IZN1_&&y&W`a|^V>FZqebC-*b&%~;cWfLgaD7=(=H7qKOwt8_E#Ntv4 z#->B_na*Y#63oIzXW#Da*^|Q!U>1oLJ&r;AB?s|Lfr)%VYZzzsFj;n}!S-}h&~WUz z%Byo{QeQZ8|Je?VFu5Z!NAl3RNHuWWVyx?ZP!L}a2Po#QH!`u{6_eM_{-(caEn4T% zyBR;ihGr!9{px!9``+m08#P47kkD}nJ|)!%>Rc(~ zov)F2MHX7*nhlrTI0_ZSfB*R-gjOer#h40O$J1584`a2uGa!4qengy^nn%><^xX{p z!z}-1z8|?90HR-TsQF)UA-?tTn>I%z!ME#kkHhE5CJV+=&Q?cq%{`*m?la7!gYa%p z=yHlAY9_pRs8=iU8NYcA1@Ynuh4QBjRuk7c-HP7xb)^n=Vah4ZX!pl~+|8jk0~3$& zMOkKP#AT?0yJW8Q=w*u z9{f2*+aqRPrDCfzBXP3m^ZnZ04-??8QMGyQx^GIO##O#rpbU2hCWE(cHoD;q1!d?g zOsF?(h~KrU_cDlWO{^2pN`E8^B7v_jgA*(g&jcw&kGcB?p>>lKHKWJb>__hGixH{E zG6~>N@%}lSwAL7_IH;NyfAgv)ADpRoi`?`xe!R(%zv{uNR^HcoyUyw>tm&2q4kNXm z;ug1U&@A}2+6lDuAb#|?cs+ckr&C4}QeO}NGn^9v-1M}dmXOk2#)87G;+(^3Xa{CK zmz3Y%mZ$J{BYJ9XzH`sA*y2{+ZPf(qk6hH<;J#duIZCJA_Sf+ViF9Xs$9zn)38Q0a z!poA?B+#*X#N}D^M zD=>0y3#oQoFOaC>Pex|=HL1^k<8SaI)%3h+d;HV+f<9;of=W2J@2ZS&LCC7a9@6nEGKku7spJ^>A!dVNky=Z!=GV<;xtB4l6VQ#4z zY&W{AYP_&u)_l`pui1Eude#t4mAfV0(MP@>7SqfUrJ3>V{KGCX`o;xkEy~QZx`8k( zk}#fS*2B!#VB$58Q^dp$rY+|WR<%3eWW8I2%xX>V?sY(Jo-;+jul5-|roH|^WDOb* zcg=Fa{z8p|tSmO(eNfJh8sxKqEdEK_3RyLW#Q0D*wCJL?mN1lEpMk68J={5X$p|)z zKff9ZTO)_QKJ25pHd*3+l1YMptY2s?Oq%~?M<}OtMSnE0!T9~2DRM)2h8^a+_^t^7 z5FoKU1G2On`P8+6P5Mt^1Jal4Cnx1-L6o25=>iS})l7)`RWWA1;#O#}o`kB9yMA~@ zqpwyWC_e4G$PZ(#gleKHR!OTu$+`R6*XEe|Z1BkiSNqmKt2&(_!E%0m+0W;ZJ|4{V znQKtSg&)eqWvE9N#p#7c%9n`SPFn%i7YO^kvwp_zf-4Iwy&@10#|=98rq+z%AqMlY zewOLBOQK{i3r}M|kDt%`N~bxcthUdGjM^h-_nK_u4bM$cGyg_m-f4s#>u7zwT)U6esG@-b%3eLj~W>VH}Sz^I^a)tHFuF zFVx0;Vw1chySWr&dmfXTF=DMTZ*v%*;iVr2f;87quX8838C3m$svw>AS0|gz`xp}* zn&AsZDkg`_XhQ^|%FXn49#t%UNJoxGNa47w4f4pES^J)A&~+2A$vl)m|My|g}&S+$Q!#}PJdpS*rrH+wDc@$m;%yRV8!)6qS~ znp}$XT1AT|9wm{5iG_QKcR%otBP$&${mF_(|Ge}+UkMxd?$~d~_IXfOV!G?ygvVv> z0RgYov?`!++^t2NK)g-UGXaM@fnRVi*dJhG4zO@9tIT;v8w+~*Ner7yfPxHp0y6_N zv_luSZYU9-H}nhFWT-a1kw!3;($MAz8l`tG-O zdwF{@q8k4wNtLxG5U63v*~((yVhCX+zzhs-M%%{t$grIg8J$yi>*~ ztKOaGE#{K1gV@_fE-R~`_j&vL!t)f*wHC&VQ?oQ&vIleNivvWdIMy?a8Ime`y$I^y%G-y{{Y^OmYXK6lClzK#sLIK*P;i)dBELmR@@|p0tcn zDr7!r;NmrOIIu=-wl_4obyq{w_HG=V?Z#sftIoP-4cPIQ1RThDzB|jPY;ktXGp<{t zb>m7EEs82#A%`-xAB~%GYc-AjGp09G%d<46@`I=lNga^EkhHMq^s+5SPb3* zet{om#sWOwy=_C$f46!hV!2GR%%4-D`qBKxlm}YmL`$9T)lXrhk{cpus;p)i(=1WM zsZyVVHf;=J9m}Gw^ESs^XO`rsHGya#TerEW$R6N`x@3%F{Vy%g$+1f6|x` zrJQV=@HJFp9i6yL!}kO`kN~CIEn2I5z-lAHEQ))glh)GsRbvHxGYeX9QbMgswQCbD zi_>*1W=!$t5S&U*S*n2nBDU)+X#@(j^xIN`$F#e^T5^Oenw^gWZgNS*re0`IaPwH= z6CO32ynR6u$9oTlTQ9NfqQ36}kc-BB@btUo%@lN)bNN+I?J7@UNB$~>R9WOBYq0NZ zvr+;|&R}R>dS|Vsf`P5A8J)GEyC(FuvKQa;JvNccpXZ;4-bX!-mQ_=9nUb32Yj@dw z`0CA! 0, Total - symsum(R(h), h, 0, height)); +coffer(height) = piecewise(height < 0, Total, height >= 0, coffer(height-1) - R(height)); +R(height) = coffer(height-1)/Total * IV; +syms x +r = R(x); + +fplot(r, [0, 5]); +fibonacci \ No newline at end of file diff --git a/lib/rlepluslazy/bits.go b/lib/rlepluslazy/bits.go index b3b0061b891..6cf57e27bd7 100644 --- a/lib/rlepluslazy/bits.go +++ b/lib/rlepluslazy/bits.go @@ -129,6 +129,23 @@ func (it *it2r) init() error { return nil } +func SliceFromRuns(source RunIterator) ([]uint64, error) { + rit, err := BitsFromRuns(source) + if err != nil { + return nil, err + } + + res := make([]uint64, 0) + for rit.HasNext() { + bit, err := rit.Next() + if err != nil { + return nil, err + } + res = append(res, bit) + } + return res, nil +} + func RunsFromBits(source BitIterator) (RunIterator, error) { it := &it2r{source: source} @@ -137,3 +154,7 @@ func RunsFromBits(source BitIterator) (RunIterator, error) { } return it, nil } + +func RunsFromSlice(slice []uint64) (RunIterator, error) { + return RunsFromBits(BitsFromSlice(slice)) +} diff --git a/lib/rlepluslazy/bitvec.go b/lib/rlepluslazy/bitvec.go index 9bca4a399a1..e442c09c43a 100644 --- a/lib/rlepluslazy/bitvec.go +++ b/lib/rlepluslazy/bitvec.go @@ -10,18 +10,18 @@ type rbitvec struct { } func readBitvec(vec []byte) *rbitvec { - bv := &rbitvec{vec: vec} + bv := &rbitvec{ + vec: vec, + index: 1, + bitCap: 8, + } if len(vec) > 0 { bv.bits = uint16(bv.vec[0]) } return bv } -const ( - minCap = 8 - maxCap = 16 -) - +// bitMasks is a mask for selecting N first bits out of a byte var bitMasks = [9]byte{ 0x0, 0x1, @@ -35,54 +35,80 @@ var bitMasks = [9]byte{ } func (bv *rbitvec) Get(count byte) byte { - res := byte(bv.bits) & bitMasks[count] - bv.bits = bv.bits >> count - bv.bitCap = bv.bitCap - count + res := byte(bv.bits) & bitMasks[count] // select count bits + bv.bits = bv.bits >> count // remove those bits from storage + bv.bitCap = bv.bitCap - count // decrease nuber of stored bits - if bv.index < len(bv.vec) { + if bv.index < len(bv.vec) { // if vector allows + // add bits onto the end of temporary storage bv.bits = bv.bits | uint16(bv.vec[bv.index])<> 7 // inc == 1 iff bitcap<8 (+10% perf) - bv.index = bv.index + int(inc) - bv.bitCap = bv.bitCap + inc<<3 + // This is equivalent to + // if bv.bitCap < 8 { + // bv.index++ + // bv.bitCap = bv.bitCap + 8 + // } + // but implemented without branches because the branch here is unpredictable + // Why this is without branches and reading has branch? + // Because branch above is predictable, in 99.99% of cases it will be true + + // if bitCap < 8 it underflows, then high bits get set to 1s + // we shift by 7 so the highest bit is in place of the lowest + inc := (bv.bitCap - 8) >> 7 // inc == 1 iff bitcap<8 (+10% perf) + bv.index = bv.index + int(inc) // increase index if we need more bits + bv.bitCap = bv.bitCap + inc*8 // increase bitCap by 8 return res } func writeBitvec(buf []byte) *wbitvec { + // reslice to 0 length for consistent input but to keep capacity return &wbitvec{buf: buf[:0]} } type wbitvec struct { - buf []byte - index int + buf []byte // buffer we will be saving to + index int // index of at which the next byte will be saved - bits uint16 - bitCap byte + bits uint16 // temporary storage for bits + bitCap byte // number of bits stored in temporary storage } func (bv *wbitvec) Out() []byte { if bv.bitCap != 0 { + // if there are some bits in temporary storage we need to save them bv.buf = append(bv.buf, 0)[:bv.index+1] bv.buf[bv.index] = byte(bv.bits) } if bv.bitCap > 8 { + // if we store some needed bits in second byte, save them also bv.buf = append(bv.buf, byte(bv.bitCap>>8)) + bv.index++ + bv.bits = bv.bits - 8 } return bv.buf } func (bv *wbitvec) Put(val byte, count byte) { + // put val into its place in bv.bits bv.bits = bv.bits | uint16(val)< cap(bv.buf) { + bv.buf = append(bv.buf, 0) + } + bv.buf = bv.buf[:bv.index+1] + // save the bits bv.buf[bv.index] = byte(bv.bits) // Warning, dragons again - inc := (^(bv.bitCap - 8)) >> 7 // inc == 1 iff bitcap>=8 - bv.index = bv.index + int(inc) - bv.bitCap = bv.bitCap - inc<<3 - bv.bits = bv.bits >> (inc << 3) + // if bitCap is greater than 7 it underflows, same thing as in Put + inc := (7 - bv.bitCap) >> 7 // inc == 1 iff bitcap>=8 + bv.index = bv.index + int(inc) // increase index for the next save + bv.bitCap = bv.bitCap - inc*8 // we store less bits now in temporary buffer + bv.bits = bv.bits >> (inc * 8) // we can discard those bits as they were saved } diff --git a/lib/rlepluslazy/bitvec.go.bak b/lib/rlepluslazy/bitvec.go.bak new file mode 100644 index 00000000000..da713bc432d --- /dev/null +++ b/lib/rlepluslazy/bitvec.go.bak @@ -0,0 +1,63 @@ +package rlepluslazy + +type rbitvec struct { + index int + + bits uint64 + bitCap byte + + vec []byte +} + +func readBitvec(vec []byte) *rbitvec { + bv := &rbitvec{vec: vec} + for n := 7; n >= 0; n-- { + var o uint64 + if len(bv.vec) > n { + o = uint64(bv.vec[n]) + } + + bv.bits = bv.bits<<8 | o + bv.index++ + } + bv.bitCap = 64 + return bv +} + +const ( + minCap = 8 +) + +var bitMasks = [9]byte{ + 0x0, + 0x1, + 0x3, + 0x7, + 0xF, + 0x1F, + 0x3F, + 0x7F, + 0xFF, +} + +func (bv *rbitvec) Get(count byte) byte { + res := byte(bv.bits) & bitMasks[count] + bv.bits = bv.bits >> count + bv.bitCap = bv.bitCap - count + + if bv.bitCap < minCap { + var add uint64 + for n := 6; n >= 0; n-- { + var o uint64 + if len(bv.vec) > bv.index+n { + o = uint64(bv.vec[bv.index+n]) + } + + add = add<<8 | o + } + bv.index = bv.index + 7 + bv.bits = bv.bits | add<<(bv.bitCap) + bv.bitCap = bv.bitCap + 7*8 + } + return res +} diff --git a/lib/rlepluslazy/bitvec_test.go b/lib/rlepluslazy/bitvec_test.go new file mode 100644 index 00000000000..6938bd7932e --- /dev/null +++ b/lib/rlepluslazy/bitvec_test.go @@ -0,0 +1,21 @@ +package rlepluslazy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadBitVec(t *testing.T) { + buf := []byte{0x0, 0xff} + bv := readBitvec(buf) + + o := bv.Get(1) + assert.EqualValues(t, 0, o) + + o = bv.Get(8) + assert.EqualValues(t, 0x80, o) + + o = bv.Get(7) + assert.EqualValues(t, 0x7f, o) +} diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go index b8f442ce08b..c72ae20c7dd 100644 --- a/lib/rlepluslazy/rleplus.go +++ b/lib/rlepluslazy/rleplus.go @@ -22,9 +22,14 @@ type RLE struct { type change struct { set bool + reset bool index uint64 } +func (c change) valid() bool { + return c.reset || c.set +} + func FromBuf(buf []byte) (*RLE, error) { rle := &RLE{buf: buf} @@ -36,7 +41,131 @@ func FromBuf(buf []byte) (*RLE, error) { } func (rle *RLE) RunIterator() (RunIterator, error) { - return DecodeRLE(rle.buf) + source, err := DecodeRLE(rle.buf) + return source, err +} + +/* + if err != nil { + return nil, err + } + + if len(rle.changes) == 0 { + return source, nil + } + + ch := rle.changes + // using stable sort because we want to preserve change order + sort.SliceStable(ch, func(i int, j int) bool { + return ch[i].index < ch[j].index + }) + for i := range ch { + if i+1 >= len(ch) { + break + } + if ch[i].index == ch[i+1].index { + ch[i].set = false + ch[i].reset = false + } + } + + sets := make([]uint64, 0, len(ch)/2) + clears := make([]uint64, 0, len(ch)/2) + for _, c := range ch { + if c.set { + sets = append(sets, c.index) + } else if c.reset { + clears = append(clears, c.index) + } + } + + setRuns, err := RunsFromSlice(sets) + if err != nil { + return nil, err + } + clearRuns, err := RunsFromSlice(clears) + if err != nil { + return nil, err + } + + it := &chit{source: source, sets: setRuns, clears: clearRuns} + return nil, nil +} + +type chit struct { + source RunIterator + sets RunIterator + clears RunIterator + + next Run + + nextSource Run + nextSet Run + nextClear Run +} + +func (it *chit) prep() error { + var err error + if !it.nextSource.Valid() && it.source.HasNext() { + it.nextSource, err = it.source.NextRun() + if err != nil { + return err + } + } + + if !it.nextSet.Valid() && it.sets.HasNext() { + it.nextSet, err = it.sets.NextRun() + if err != nil { + return err + } + } + + if !it.nextClear.Valid() && it.clears.HasNext() { + it.nextClear, err = it.clears.NextRun() + if err != nil { + return err + } + } + + for len(it.changes) != 0 && !it.changes[0].valid() { + it.changes = it.changes[1:] + } + + var ch change + if len(it.changes) != 0 { + ch = it.changes[0] + } + + if it.source.HasNext() { + var err error + it.nextRun, err = it.source.NextRun() + if err != nil { + return err + } + } + + if ch.valid() && ch.index < it.runIndex+it.nextRun.Len { + if ch.set != it.nextRun.Val { + // split run + whole := it.nextRun + it.nextRun.Len = ch.index - it.runIndex + + } + + // if we are here then change was valid so len(it.changes) != 0 + it.changes = it.changes[1:] + } else { + it.runIndex = it.runIndex + it.nextRun.Len + } + return nil +} + +func (it *chit) HasNext() bool { + return it.nextRun.Valid() +} + +func (it *chit) NextRun() (Run, error) { + return it.nextRun, it.prep() } func (rle *RLE) Set(index uint64) { @@ -44,5 +173,10 @@ func (rle *RLE) Set(index uint64) { } func (rle *RLE) Clear(index uint64) { - rle.changes = append(rle.changes, change{set: false, index: index}) + rle.changes = append(rle.changes, change{reset: true, index: index}) +} + +func (rle *RLE) Merge(other *RLE) RunIterator { + return nil } +*/ diff --git a/lib/rlepluslazy/rleplus_test.go b/lib/rlepluslazy/rleplus_test.go index 38746c755b3..3b47d02c2df 100644 --- a/lib/rlepluslazy/rleplus_test.go +++ b/lib/rlepluslazy/rleplus_test.go @@ -4,7 +4,7 @@ import ( "math/rand" "testing" - "github.com/filecoin-project/go-lotus/extern/rleplus" + "github.com/filecoin-project/lotus/extern/rleplus" "github.com/stretchr/testify/assert" ) @@ -151,20 +151,20 @@ func BenchmarkOldRLE(b *testing.B) { func BenchmarkDecodeEncode(b *testing.B) { b.ReportAllocs() var r uint64 + out := make([]byte, 0, len(goldenRLE)) + for i := 0; i < b.N; i++ { + rle, _ := FromBuf(goldenRLE) + rit, _ := rle.RunIterator() + out, _ = EncodeRuns(rit, out) + r = r + uint64(len(out)) + } + /* - out := make([]byte, 0, len(goldenRLE)) for i := 0; i < b.N; i++ { - rle, _ := FromBuf(goldenRLE) - rit, _ := rle.RunIterator() - out, _ = EncodeRuns(rit, out) + rle, _ := rleplus.Decode(goldenRLE) + out, _, _ := rleplus.Encode(rle) r = r + uint64(len(out)) } */ - - for i := 0; i < b.N; i++ { - rle, _ := rleplus.Decode(goldenRLE) - out, _, _ := rleplus.Encode(rle) - r = r + uint64(len(out)) - } Res = Res + r } diff --git a/lib/rlepluslazy/runs.go b/lib/rlepluslazy/runs.go new file mode 100644 index 00000000000..c29e0002ff7 --- /dev/null +++ b/lib/rlepluslazy/runs.go @@ -0,0 +1,96 @@ +package rlepluslazy + +func Sum(a, b RunIterator) (RunIterator, error) { + it := addIt{a: a, b: b} + it.prep() + return &it, nil +} + +type addIt struct { + a RunIterator + b RunIterator + + next Run + + arun Run + brun Run +} + +func (it *addIt) prep() error { + var err error + + fetch := func() error { + if !it.arun.Valid() && it.a.HasNext() { + it.arun, err = it.a.NextRun() + if err != nil { + return err + } + } + + if !it.brun.Valid() && it.b.HasNext() { + it.brun, err = it.b.NextRun() + if err != nil { + return err + } + } + return nil + } + + if err := fetch(); err != nil { + return err + } + + // one is not valid + if !it.arun.Valid() { + it.next = it.brun + it.brun.Len = 0 + return nil + } + + if !it.brun.Valid() { + it.next = it.arun + it.arun.Len = 0 + return nil + } + + if !it.arun.Val && !it.brun.Val { + min := it.arun.Len + if it.brun.Len < min { + min = it.brun.Len + } + it.next = Run{Val: it.arun.Val, Len: min} + it.arun.Len -= it.next.Len + it.brun.Len -= it.next.Len + return nil + } + + it.next = Run{Val: true} + // different vals, 'true' wins + for (it.arun.Val && it.arun.Valid()) || (it.brun.Val && it.brun.Valid()) { + min := it.arun.Len + if it.brun.Len < min && it.brun.Valid() || !it.arun.Valid() { + min = it.brun.Len + } + it.next.Len += min + if it.arun.Valid() { + it.arun.Len -= min + } + if it.brun.Valid() { + it.brun.Len -= min + } + if err := fetch(); err != nil { + return err + } + } + + return nil +} + +func (it *addIt) HasNext() bool { + return it.next.Valid() +} + +func (it *addIt) NextRun() (Run, error) { + next := it.next + return next, it.prep() +} diff --git a/lib/rlepluslazy/runs_test.go b/lib/rlepluslazy/runs_test.go new file mode 100644 index 00000000000..3ab86e76f19 --- /dev/null +++ b/lib/rlepluslazy/runs_test.go @@ -0,0 +1,94 @@ +package rlepluslazy + +import ( + "math/rand" + "sort" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSumRuns(t *testing.T) { + { + a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14}) + assert.NoError(t, err) + b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70}) + assert.NoError(t, err) + + s, err := Sum(a, b) + assert.NoError(t, err) + bis, err := SliceFromRuns(s) + assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis) + assert.NoError(t, err) + } + + { + a, err := RunsFromSlice([]uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14}) + assert.NoError(t, err) + b, err := RunsFromSlice([]uint64{0, 1, 2, 3, 9, 10, 16, 17, 18, 50, 51, 70}) + assert.NoError(t, err) + + s, err := Sum(b, a) + assert.NoError(t, err) + bis, err := SliceFromRuns(s) + assert.Equal(t, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 50, 51, 70}, bis) + assert.NoError(t, err) + } +} + +func randomBits(N int, max uint64) []uint64 { + all := make(map[uint64]struct{}) + for len(all) <= N { + x := rand.Uint64() % max + if _, has := all[x]; has { + continue + } + all[x] = struct{}{} + } + + res := make([]uint64, 0, N) + for x := range all { + res = append(res, x) + } + sort.Slice(res, func(i, j int) bool { return res[i] < res[j] }) + + return res +} + +func sum(a, b []uint64) []uint64 { + all := make(map[uint64]struct{}) + for _, x := range a { + all[x] = struct{}{} + } + for _, x := range b { + all[x] = struct{}{} + } + res := make([]uint64, 0, len(all)) + for x := range all { + res = append(res, x) + } + sort.Slice(res, func(i, j int) bool { return res[i] < res[j] }) + + return res +} + +func TestSumRandom(t *testing.T) { + N := 100 + + for i := 0; i < N; i++ { + abits := randomBits(1000, 2000) + bbits := randomBits(1000, 2000) + sumbits := sum(abits, bbits) + + a, err := RunsFromSlice(abits) + assert.NoError(t, err) + b, err := RunsFromSlice(bbits) + assert.NoError(t, err) + + s, err := Sum(b, a) + assert.NoError(t, err) + bis, err := SliceFromRuns(s) + assert.NoError(t, err) + assert.Equal(t, sumbits, bis) + } +} From 32a1940c576c69c56228e4176ad88baec9738a69 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 7 Dec 2019 15:47:36 +0100 Subject: [PATCH 09/12] Replace rleplus with rlepluslazy License: MIT Signed-off-by: Jakub Sztandera --- chain/types/bitfield.go | 86 +++++++++++++++++++++++++----------- lib/rlepluslazy/rleplus.go | 42 +++++++++++------- lib/rlepluslazy/runs.go | 24 ++++++++++ lib/rlepluslazy/runs_test.go | 4 +- 4 files changed, 114 insertions(+), 42 deletions(-) diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index 617229c376c..1f46e9451a6 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -3,19 +3,24 @@ package types import ( "fmt" "io" - "sort" - "github.com/filecoin-project/lotus/extern/rleplus" + rlepluslazy "github.com/filecoin-project/lotus/lib/rlepluslazy" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" ) type BitField struct { + rle rlepluslazy.RLE + bits map[uint64]struct{} } func NewBitField() BitField { - return BitField{bits: make(map[uint64]struct{})} + rle, _ := rlepluslazy.FromBuf([]byte{}) + return BitField{ + rle: rle, + bits: make(map[uint64]struct{}), + } } func BitFieldFromSet(setBits []uint64) BitField { @@ -26,31 +31,59 @@ func BitFieldFromSet(setBits []uint64) BitField { return res } +func (bf BitField) sum() (rlepluslazy.RunIterator, error) { + if len(bf.bits) == 0 { + return bf.rle.RunIterator() + } + + a, err := bf.rle.RunIterator() + if err != nil { + return nil, err + } + slc := make([]uint64, 0, len(bf.bits)) + for b := range bf.bits { + slc = append(slc, b) + } + + b, err := rlepluslazy.RunsFromSlice(slc) + if err != nil { + return nil, err + } + + res, err := rlepluslazy.Sum(a, b) + if err != nil { + return nil, err + } + return res, nil +} + // Set ...s bit in the BitField func (bf BitField) Set(bit uint64) { bf.bits[bit] = struct{}{} } -// Clear ...s bit in the BitField -func (bf BitField) Clear(bit uint64) { - delete(bf.bits, bit) -} - -// Has checkes if bit is set in the BitField -func (bf BitField) Has(bit uint64) bool { - _, ok := bf.bits[bit] - return ok +func (bf BitField) Count() (uint64, error) { + s, err := bf.sum() + if err != nil { + return 0, err + } + return rlepluslazy.Count(s) } // All returns all set bits, in random order -func (bf BitField) All() []uint64 { - res := make([]uint64, 0, len(bf.bits)) - for i := range bf.bits { - res = append(res, i) +func (bf BitField) All() ([]uint64, error) { + + runs, err := bf.sum() + if err != nil { + return nil, err } - sort.Slice(res, func(i, j int) bool { return res[i] < res[j] }) - return res + res, err := rlepluslazy.SliceFromRuns(runs) + if err != nil { + return nil, err + } + + return res, err } func (bf BitField) MarshalCBOR(w io.Writer) error { @@ -59,7 +92,12 @@ func (bf BitField) MarshalCBOR(w io.Writer) error { ints = append(ints, i) } - rle, _, err := rleplus.Encode(ints) // Encode sorts internally + s, err := bf.sum() + if err != nil { + return err + } + + rle, err := rlepluslazy.EncodeRuns(s, []byte{}) if err != nil { return err } @@ -88,19 +126,17 @@ func (bf *BitField) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("expected byte array") } - rle := make([]byte, extra) - if _, err := io.ReadFull(br, rle); err != nil { + buf := make([]byte, extra) + if _, err := io.ReadFull(br, buf); err != nil { return err } - ints, err := rleplus.Decode(rle) + rle, err := rlepluslazy.FromBuf(buf) if err != nil { return xerrors.Errorf("could not decode rle+: %w", err) } + bf.rle = rle bf.bits = make(map[uint64]struct{}) - for _, i := range ints { - bf.bits[i] = struct{}{} - } return nil } diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go index c72ae20c7dd..6bf8f378968 100644 --- a/lib/rlepluslazy/rleplus.go +++ b/lib/rlepluslazy/rleplus.go @@ -16,25 +16,18 @@ var ( type RLE struct { buf []byte - - changes []change } -type change struct { - set bool - reset bool - index uint64 -} - -func (c change) valid() bool { - return c.reset || c.set -} - -func FromBuf(buf []byte) (*RLE, error) { - rle := &RLE{buf: buf} +func FromBuf(buf []byte) (RLE, error) { + rle := RLE{buf: buf} if len(buf) > 0 && buf[0]&3 != Version { - return nil, xerrors.Errorf("could not create RLE+ for a buffer: %w", ErrWrongVersion) + return RLE{}, xerrors.Errorf("could not create RLE+ for a buffer: %w", ErrWrongVersion) + } + + _, err := rle.Count() + if err != nil { + return RLE{}, err } return rle, nil @@ -45,7 +38,26 @@ func (rle *RLE) RunIterator() (RunIterator, error) { return source, err } +func (rle *RLE) Count() (uint64, error) { + it, err := rle.RunIterator() + if err != nil { + return 0, err + } + return Count(it) +} + /* + +type change struct { + set bool + reset bool + index uint64 +} +func (c change) valid() bool { + return c.reset || c.set +} + +func (rle *RLE) RunIterator() (RunIterator, error) { if err != nil { return nil, err } diff --git a/lib/rlepluslazy/runs.go b/lib/rlepluslazy/runs.go index c29e0002ff7..c8ae36644bf 100644 --- a/lib/rlepluslazy/runs.go +++ b/lib/rlepluslazy/runs.go @@ -1,5 +1,11 @@ package rlepluslazy +import ( + "math" + + "golang.org/x/xerrors" +) + func Sum(a, b RunIterator) (RunIterator, error) { it := addIt{a: a, b: b} it.prep() @@ -94,3 +100,21 @@ func (it *addIt) NextRun() (Run, error) { next := it.next return next, it.prep() } + +func Count(ri RunIterator) (uint64, error) { + var count uint64 + + for ri.HasNext() { + r, err := ri.NextRun() + if err != nil { + return 0, err + } + if r.Val { + if math.MaxUint64-r.Len > count { + return 0, xerrors.New("RLE+ overflows") + } + count += r.Len + } + } + return count, nil +} diff --git a/lib/rlepluslazy/runs_test.go b/lib/rlepluslazy/runs_test.go index 3ab86e76f19..aefeb2c708e 100644 --- a/lib/rlepluslazy/runs_test.go +++ b/lib/rlepluslazy/runs_test.go @@ -76,8 +76,8 @@ func TestSumRandom(t *testing.T) { N := 100 for i := 0; i < N; i++ { - abits := randomBits(1000, 2000) - bbits := randomBits(1000, 2000) + abits := randomBits(1000, 1500) + bbits := randomBits(1000, 1500) sumbits := sum(abits, bbits) a, err := RunsFromSlice(abits) From 6e4761c819f7ba862aeb4c80baeeecdf674d9923 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 7 Dec 2019 15:51:31 +0100 Subject: [PATCH 10/12] Fix rleplus in actor code , License: MIT Signed-off-by: Jakub Sztandera --- chain/actors/actor_miner.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 0ae52a059ae..70eed93de5d 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -477,7 +477,10 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.Absorb(err, 3, "could not decode sectorset") } - faults := self.CurrentFaultSet.All() + faults, nerr := self.CurrentFaultSet.All() + if nerr != nil { + return nil, aerrors.Absorb(err, 5, "RLE+ invalid") + } _ = faults proverID := vmctx.Message().To // TODO: normalize to ID address From 80dbc25b6d2e9919a725974ca0d4a1a8052e6991 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 7 Dec 2019 15:56:44 +0100 Subject: [PATCH 11/12] Fix typo in overflow check License: MIT Signed-off-by: Jakub Sztandera --- lib/rlepluslazy/runs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rlepluslazy/runs.go b/lib/rlepluslazy/runs.go index c8ae36644bf..be2ed5bcc99 100644 --- a/lib/rlepluslazy/runs.go +++ b/lib/rlepluslazy/runs.go @@ -110,7 +110,7 @@ func Count(ri RunIterator) (uint64, error) { return 0, err } if r.Val { - if math.MaxUint64-r.Len > count { + if math.MaxUint64-r.Len < count { return 0, xerrors.New("RLE+ overflows") } count += r.Len From 71b973f8307546865e1b0a5d63cd6045e662861b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 7 Dec 2019 16:19:54 +0100 Subject: [PATCH 12/12] Some cleanup License: MIT Signed-off-by: Jakub Sztandera --- chain/types/bitfield.go | 7 +- docs/rewards.m | 29 ------ docs/rewards.m~ | 30 ------ docs/rewards1.jpg | Bin 27804 -> 0 bytes docs/rewards2.m | 19 ---- lib/rlepluslazy/bitvec.go.bak | 63 ------------- lib/rlepluslazy/rleplus.go | 147 ------------------------------ lib/rlepluslazy/rleplus_reader.go | 10 +- lib/rlepluslazy/runs.go | 2 +- 9 files changed, 13 insertions(+), 294 deletions(-) delete mode 100644 docs/rewards.m delete mode 100644 docs/rewards.m~ delete mode 100644 docs/rewards1.jpg delete mode 100644 docs/rewards2.m delete mode 100644 lib/rlepluslazy/bitvec.go.bak diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index 1f46e9451a6..b4687f5e8ff 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -16,7 +16,10 @@ type BitField struct { } func NewBitField() BitField { - rle, _ := rlepluslazy.FromBuf([]byte{}) + rle, err := rlepluslazy.FromBuf([]byte{}) + if err != nil { + panic(err) + } return BitField{ rle: rle, bits: make(map[uint64]struct{}), @@ -70,7 +73,7 @@ func (bf BitField) Count() (uint64, error) { return rlepluslazy.Count(s) } -// All returns all set bits, in random order +// All returns all set bits func (bf BitField) All() ([]uint64, error) { runs, err := bf.sum() diff --git a/docs/rewards.m b/docs/rewards.m deleted file mode 100644 index 28dbe476015..00000000000 --- a/docs/rewards.m +++ /dev/null @@ -1,29 +0,0 @@ - -HalvingPeriod = vpa(6 * 365 * 24 * 60 * 2); -AdjusmentPeriod = vpa(20160); -Decay = exp(log(vpa(0.5)) / HalvingPeriod * AdjusmentPeriod); -Total = vpa(1400e6); -IV = Total * (1-Decay) / AdjusmentPeriod; - - -syms R(x) -R(x) = IV * (Decay.^floor(x/AdjusmentPeriod)); - -Hmax = HalvingPeriod*5; -heights = linspace(0, Hmax, Hmax/AdjusmentPeriod); -Rewards = R(heights); - -R2 = zeros(size(heights)); - -coffer = Total; -for h = 1:size(heights,2) - k = IV*(coffer/Total); - coffer = coffer - k*AdjusmentPeriod; - R2(h) = k; -end -hYears = heights/2/60/24/365; -plot(hYears, Rewards, 'go', hYears, R2, 'r-') -legend('formula', 'incremental'); - -for6y = Rewards(1:HalvingPeriod/AdjusmentPeriod)*AdjusmentPeriod; -inc6t = R2(1:HalvingPeriod/AdjusmentPeriod)*AdjusmentPeriod; diff --git a/docs/rewards.m~ b/docs/rewards.m~ deleted file mode 100644 index 2f2f57c3776..00000000000 --- a/docs/rewards.m~ +++ /dev/null @@ -1,30 +0,0 @@ - -HalvingPeriod = vpa(6 * 365 * 24 * 60 * 2); -AdjusmentPeriod = vpa(20160); -Decay = exp(log(vpa(0.5)) / HalvingPeriod * AdjusmentPeriod); -Total = vpa(1400e6); -IV = Total * (1-Decay) / AdjusmentPeriod; - - - -syms R(x) -R(x) = IV * (Decay.^floor(x/AdjusmentPeriod)); - -Hmax = HalvingPeriod*10; -heights = linspace(0, Hmax, HalvingPeriod/AdjusmentPeriod); -Rewards = R(heights); - -plot(heights/2/60/24/365, Rewards, 'r-') - -R2 = zeros(size(heights)); - -coffer = Total; -for h = 1:size(heights,2) - - k = IV*(coffer/Total); - coffer = coffer - k*AdjusmentPeriod; - R2(h) = k; - -end -hYears = heights/2/60/24/365; -plot(hYears, Rewards, 'r-', hYears, R2, 'g-') diff --git a/docs/rewards1.jpg b/docs/rewards1.jpg deleted file mode 100644 index 7fd4bd73e764ecd24348108074d1b28c863853e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27804 zcmeHv2Ut`|*7ijZP?8`>4hjg8vj`0>B3VE{a*~`ua;7nWk|j%!jAU9QBT;hBIp>Vz z*mVExhVS1&cV~9KnVtFnXWOUjhI^^Lr|Q%>?>Y6>L5v_~fNRp?QsMv#3IL!W{{aX% zKq4#hemoYIgFs@v|yoybTgN=oSO-gVbkC2j#nu?N)f`W#Qje+JC3oQi&j_6t!9;A5iRMx{bUxdmLp zM?u9$K{No=NC7XS{N)Aw?S*m)740%Q2IiHkSjZcSuK|}(P*E?TphK9V`6L}p# z!@o>$lT!r!x`H0YEn7ma=b_1%v=0jEh!lIb>A3anUR=3)gP4SrjQ%zQ;~gd*UcS5h z_XI>Aiit}|N=Yj{QC3k^Q`aysG%_|ZH8Zz&aCCBZadmt7%E$Nh8$bWB@QBE$=yx%( zDXD4cA2KqtvOg6T6_=Ejl~;VJZ)j|4ZfR}n>mL{#8Xg%Po1L3qSX^3OSzX)N-P=Dn zJUTu({U#R*fcjUlkpKTxunThGBjvh;hK7oU@l7t2OU}rKijQ{rCMP<9hysS5?e$w+ z&oK!fgeDi%U7_Vx+$Peu>%Dq|j%SvB=bLDMk?fZV_TpDb_CJFCU%AErY*ZBFzWW^%lcvYSx{K;DmmB;lons0IOvXXGa z{2R_x{UIxOsR&?ZNEa@Vx~X#q0VD~BAb^+ZkP|9zzLi}0VwViK_bC{TV}t+}+=K_` z5>HWA5Wq2SGy(`+LACy)I?2ge7pCni%IOsMB$hU{qHQhr;Ytk{4(9{h|EH9e)^ z;%ka*W)kjbe2N0Y4YAZ}ENS^PUYhHMC zU8VX@Gc^|n{#PkUj&tsDaMx1P6&`KI;*BNDwzW@BIf52i5rD+%n{FuGV+`DwPnxeNSoxfc^;q>_cE=?FgV{AEoq9){VWQg)nW1a%q2XK)P#x9m`t1a=5A57%vS1NImGt2}y=oT@bqm zeUixUg6I+##RjZ;)e?Uy`k$(s0~H~Y20QaovS+;xPyx(a@87L`vSOx{Ov2muD)%cq zp!WX%b;3V0M1`p)$A)Gb%hXl+r_3jz>++$Kc0i--Guc{ zhxAfBWiBQ^SXAYofAQ6$lE-V;Sh#%|3&l9#p+8YKaxC`;kMg%iBbsU>5o}1Y^^A1csYj1@WGeGr#)j9jVNm z0%_?xts8;R%}xopkE{V4J_gx21()(tpAzwgoSm5=fabO7L{}@JiisjFmamI*JWL87 z$Y=GIG_gpY(0= zVm?@Ivqn;~|jwH8RfAs|h@F=xDX8Tf z*LXmCk952gc=@>TVzVWkF9Zo0H^vr-ioX(6-vd~jUra<-Mc*ZC=4mM z5P*ozZQARKQ39MclTQ1?@*O*zM9VT{PzH-;x%m9mstV~zw}c!qLe&y?Ju$+^DF5lE z7P}1d1#u$)Jq85OC*wb!D{BuIr6NTDAHxyADt#kh)yI{SEb+Ll9u5Dkn2Ee_Nnu%O z|NhBMj0IQr_aq_DZt}xAB@RQbA=enY4AKopB*OI5TdhvXSwOLqkQtigv=W)-c=%w` zz9@24?E){Ef5c9gdnV4$&l};vomL!w!*@#l775NL_QczdBuK>tKegA+Eaj`WW`eA3 ztn!kdd5QVXremy0vw>-~8Z-RaqaT$-Xh_hR>6th)w?C5!<>q?J@s7@_%cZ;2LT*8t zjjdxkT%i6wvbb|=LO%34DgG_{ zp}fW}#mc#HV>ItiUyjjA%9+V$i< zFDOl74K&%we%Yi-FkZi4=Bt@%!?ODzx7j)RW?(kogyCCPOOz(JZIodJH9V;H2Pl1c z(d&(Z>GU{oyz~1VW?ximfsZ8JHI@{aFkJ{yL72Oy>(xnZ+E>mk!GfunGj~t} zsLV~NXhX|);=KEG@@!D94^9rW+)D53DVk&Rd%zp7_%i+p9orp(-Biy-(3n&g?S$vUYfGwQ1;`XMih76PD}fYI9$zv0qB5UnQ!O zK}pN=t0QwP2VCab_o}xkP=`{5rGiPi zz1iCCcl5HgQ;4fAL{d4Eg5p(`#Q=L&+iK%T7?ww0Q2_V78xNxuycA6JaG zZS8spSD4s!bSO4NvDh`h&%z|pPUkAx7*P{mDkO|arM;|+jh7d+iA=TN<>CX{@xcrL zcET5^i*=W$;FmG6pf;bfl25**Pbg+10a{wROF)Jq zZ$ZeFKv09UrR6!+zdRwc7w{gu3C8ibYiIXR!xFQ7qnELJ31e9&M63F>F*}@O^+q$P z=j~~0cMd;EZ~}8Qxy1P62!U~V?cq5)gUpH%*Dr!&>x1Ss*HLFX*VF4YB&5Yrr~ zqNL??b=fMi? zb+F`3DP}KVU1SO36`D$QUaE8+=3AxP%GAoMPr39An8CI6M&XV5VBt&_pguyW%=YLs zy+NtddJQk?%~tkjuN8W6Jnf{t1Z1Y>B0kADcG`9Stl6^c6q;vaT%t~%lU>>-p9MTm zIL=MHY*At}YjvqbH1nk9X=#XUe?FQkJ``4Tr;zHT?jQ+Qu<=IDhhyIdFEMYAhXLe< zO&BxMdpe=Baf6jTd2oH5n~P;aBGl)WszN(>vc%U!ZC@mZrkJ48@o~j_+IO3W%xy-lwiAS34LIXkp?dFB^V=vJ9f^&p|nCH#? zKLm;V(>W&+eLTck)QZpGf6^WpQmEF+4|`-<>&1g2e*yYoyi8Z$$B04o34dJul=3PE z`kGSPM>J}E6r!4+*9g9c1me8~Pf4yx7btZnQ-ow@DBLG3+PvK_9sICJU75})^i_D} zOM6aZT3dd)0tQNZS*sg$DV!(t5@TcwT#{|=ce>O#?vb`qIMF%qdiet{`ns^NY%hPZ z3?!H+(3heMlUfLqqDzpoVE;!iXklFdB1ZtX?!HC<8{}39ASKHC_6}v~SS8r$<-I3+ z=;&Tw!F`p(UxwR1Sj=$GTtTU&u=H34i+8&1059-p4xki6mR;|mZ-|I43Eo#rNDf}0 z3va#;COd)65oD+k&DX3I>t@X?6)lowr#V~5??_DP@|A}j2!;74r%baYr)f^Gqva1# zy?SPT?q0N%Eg~n#J)W+}OOrtvTc#jJWaRVR%K7I3+;=hNZ~V6;-x=5f5Hfv9j+dd2I>SF0);drXPNmiHShtDqr?u{((LN#)^+$Q$#furT4 zR@#}es7;s@+{9Ml#9t-0k?T=4+wJ*3T_OMZxZyXjz<<-DUy#0_O=$@XPLA}&6v9kT z@t)Y}R2j3Nv`T5GGvCCe+-Z`zd5(8}pp*4eeYOZV_WRIcBvV#kAoUy1u z1q@X8nz5G5JVs%1M;(ALopfNGmIyqd>Xsk$!svRMIJjdTzV~|as|c*$4DxDYCB^@g zdj*N;esLUm>36>Ow?rKc+cvN_Q`ATdZdD5R*BHO(ypS1lD`1)c_iV z3BY5Zg4^j|51pzr(_i>`j}J2<;kpN0c|9;WB5Oq)$E6Euda6==OCPh z$YaYELoSySA09jCMJx>A%V1PirX7SV^c+QZHpem#9J(=sA6CPAdpEa87Tb`){S*aR z57aAuQ1VW*N!FHP7THH-+Co+29$#(`V~RiUXJ`$-Mf_svGsM7|E zDSGSTcF3RG5q+rdPFmTnU}I?DN0==B!1sZi^Y!xJ$}}YC1bwyT zdG6%ve2<@$AJzLF2=~KB5YcQWE*Q&{?1kx=-BcWwt8fqNW2wrfw2}=I1|f_5AFuqY z0=;5@3j*-lKj;J9mrlmd+0x9w||CJ&g7e%yh>K&cRM{@ZAt)}R%<-XMzg9w>Y0WY0CWJNdKg>;gV zRzolk+`a<>O}*wxr9Z41KdI6?l45n|bIVn{k}Rc~nyJ^FCtTi*ZhmV|;u`l>7;13U zll}bQsz6>Nod|Qb%ma>hK5vrAa9c)Qf^-HKiEGc0b0Lkaco)a|cF%yZ4CJV}*_fiZaNZAfwL1UGd?a|e@BQ|k%3i9w6z(*KtN-glToSLubjOOR1YV(Ab0dsq zSGPh+I|LxF=XIp?Y=uMe2|kC`yJn{upiu}Q0QQdIj(MAxP1)-vP$V;=!;uk=yIJy8rdD><3OU*#|^9@Fn0Tn~HgI5(Y<{}F>=2Twd zTxX6!OVF82Goe35C48-0gjJOLWx4wX0x%jstG}szW*hH09e1w@W!WDXxafcK1er&8 zv227dNpBfGt-q`;g`MU^^MaZCACZkIl4bA@xV$o_S;JnS_}bvWet(o#fx=9y$pnT7Sb8GF3- z8;|Q*o`n*-SeKsC+&K_!ffx6yhPsF~U6MKXvVE%cqw4bH)BKO!UOHL79d zBeUy1-K}}n4X*pJP62;lKDnRwj>BEQ{S&KQaHSOPHXSLs9;~PU1DS_vG5o(l&w)3EWekX2H0?8v9>y%ncNK=k?{|h_@EH z?F8YdtH}P>|B&$ zwZ;c7Z&SYF(Mfzjp-X$Lw1@yojhu-QfOXrBO`$|auNcw(a{uq09r>62KOW#Yi)+PC z8e-4A6i8GeDQRQBB;LH+sGH)=0iq-qO9I(2ceG~e-U+S6UeK0l9Rpg|Yf(p3&VQ9< z_-lan(>UZunJ&JS3$dL4(iHP&GuEa1J1w8p$adn!U5l(7a$tF%+x5Yf_f%kzFID+v z)NeE-;y&%Ts1vvAV@%q%m7eTFi3RHS`jn$~rjg8`@PVB|fBa|4>7V*^XFVaC!)EL} zPGsUqyh3bP$@N^b&8fIJI#8iw&z$SDqa6*dsW-X5v{Z5+0O1!^S3f@^3CVq|k%-yY|7g`eYS@41^Y>G6F$8t^#b!q3ryP-?$%|=>D;L1_wtFGPwx-}j1b`)*S5?P1dD*RynHjZ(M#w=jhKlDaC)<- z9bZrB_U}XEYNcj&Z198<5*OeWqRSlIQP;0;EH7fLMVG;Ck30>QoQsf|YR>)SpR-)Q zV)7PK=d%Q02(|*Rokyv2*>zQ+c7=$ZkZrB^)#-Z`h{@-7 z4`;2+*y2_ZE~iC>X{VJUu}wwS$CL}io&V6!s23dEuOx+t8;P+rFg*{Gs=btQrd>K( z-GBI|GSH-Rif?|s&-?awm?hoF5j&Su&G4EXX0Gl9n#x}?Y2N|9)LgUnY%_a}lDizp zJlPFOypv7U%*=RNMK*%bP{nw2kuT<}D=rc2d?9(hV+w(BB_Eqci8EpM&vD#uC%RLm z>fsD%u>{1sQO%XvP_G!4j?d|%b)ZsjNn*4?r#x|0r| z@wHuQESmKLqpNWFrJ@62cML_fNAiW4M@71MRjrWmUM*;+MUD_UGH&?r$Hm|mP1~-jny5`5#Y;P7LH8&L( zF5PeG_P|TZJH4nyzpI1%JH&r-*Qr2fOvZ|%0~zuo84`K2Vz(nA6TU<|2@e=eV9!;T zNwMIvrhD}?jDu43Soh%EGiLyFH-}PFM)mDWTjExR@(l>}PA&oO6i7V^j>B+*y z+iC8S7;uE0=lE@x6yFZWB1tnb9lkn)04PjSJtg*!pzl6RX$IIv*CGJ6ki|k_V_bEw zn|mJ263gi4!|UiLy5TClR8?ZUl!jssPups*N!ZFkGlky^{t+8||M88yp9BOK;2Nbx zYo&IIRGRrMHDb;^IyK~+gs*|WR2diYZgQWgN=03!uGUSUh4&x;Dv5a=C!<%Vjg-Rl z$b=Vr%(5}RszxG+Xpy{lsiL`0MwGlxU~F0~fZ=H;!Iod)5t2fe2X|9d*STY4FYJ?> z-_i8Y=Peaq&Cbo!z2>2+0QssAg^#{#Y&XQZgz#*&PQ(jSHa9QV4x9~ArupQ z>lJffWC+i3-ZF>vPnfwBlqwT%iamt^DzWZ6aF=zJF>{Lg`XodK>BUyNKm73NemfzDw^g;X?Y9>IuG4ixasZoLqrNGqsIhXr~7fa%$oJE5ATokl!L1kUy z~^x&Dnk;>VQq5|6AU#IbX^Fk5J-M5u}-;P{^*JQHzo;qHge?bxbiXJ5Fw zpjuB@(iu?h4aWk}4+e||So5PiKUipHS-p>VD%><%FisPAwp8Fjls{VIQ-pOBH+AX! z#gtI9Tp9j?c+GClhYt5;jG+o4uzY}Q2}eWM;)1JT=uY{hA4@ZUoAD7)f&7xQ3V)B2 z{`QU?%ed?_8!0NjrW|pEk>^VMX5Y_2NbS?5Hgr{rm|NZOSNG06Ek>BRwfks?c$QxI zTxajR9(p8yOqB@F-m?c051CDY3C6oy_Uu{6^Kf05+l0U|jsXX&qieZ_F5m{;F{in{ zIWe2T05~n|C_lSW?g|?ZtGpfegNN~kkeqPP%MD24P?f#-AnoA(Lir<%vrp_t znl-apCvt*g%_9R|yW^dZ*n7LVCIW|C&g(ahEDGWKP7trGi5{f#jR_^C{A)A!Z}UHY z)L%B4Z-LbZNk5QkmzJStY?E2@oPjv%PR{;g^nbyX`p;}#{oLZa$4<6##nW)OR~-9I zSVXI{MWpv1NfG_Qsi40j-HY!}oNw%CYgS%Qa*5ZAI3l7ajFnaQOQ$y} zSP-V|$>*_pSZOZ3w|Lh~=`8IuWO>6PYum&xYKiV+d4$6_>Rnyg*VFyG`}sMF{j&ob zhQ;r&IZiZ587JRBX$KwFt1uqh(z%dM%!-TRcGV28t}!oh5}F$o4{GRQ!I5BD+@sN) zi0gDGlC>JiAx9paJ8bS<*&iQW$+xekx*eeUoF-;K+oy$Qhw*?4p1O1D;JE*U`Aa2{ zw2kP>vtxf3fAjRa84G9NZxN2cvmdRd-|%<8QGx=w`|%xpuw?xg2A!9Fgi1aYZ<6xn z;4wdwdb{Px=PgpZJP51ubNFfn?=~VOLVD?Zgp^G*zlaknBy)?EUyy+d(VDe`V$Deyy+Y zz8S+;Qfr8@BLPpG4?K+u%^6?6q3?oAVoPVZ`x+*MV|ZLxpOZDt+gZsp%*3&Z0EAR2 zIVXarqtjjCahX%5cUNyoj$oLYY=M!;85u5VnJg!(^JPU$_LFQcn)?%bD_~?N8hWZ!=8C+f-3Y3xlN-((KEkY@;vi)PzLMj$I|f4 zOM_@}9p*CDO?LJ}-U4gC1`{t;O3cc5Dv}zP1gWf+8^K9EMKR6IpnjFyR0UY_m$Ii&Uf@YdBp>aA7hH3dbWDlA-0u^=WqpO3hoL8U9q#6EQr1?j zsu{VCaeV3oiPg7vk-I!uk9;boRsR&BH~Q zyg4?r!K)Zs3(YH@HPL$`-#iCV(qBABPlgWC;%IX>T3ddFfn7W7nit6rTOtTJn>G}hn-dXSPufY(dM!51>e6X3a3h`ELHDecVM2Z zQ)Q2ufrWbs#bul{dQ3|5?zScL#Z`jys1uDnHZ>f#i9A!HLNz3 z!=3cBsMW2SPA7NHeFZVc_s@UhZKXeYE9n=TN>N9%%NsbCJ7id_2Gt<(WU>CTcbJ8I z7_->anN!WexhiaWRrnW?YXwLoTPkVprIiD5~qFGRKjb zm5o1m7H*0FY)?jW3P1E6Sjg&3w40xdJ!_fZO0-qYX{4#sQ$c3DFX6{s;T1Ry(055qM~SI2DSBsQjKL$mT~x#&BLM2L-;y; zy>P9_boDRUHpAaC+xF*%4L<<5sLbj;R=n?;W3lZSzeuNsZg`5hMitDiZ{lu5*5b9$ z3O`AlnO=@PWHS8N43eej%?&M8)_ZHw(X<$`BfQ(EeRi5)%+2a!yo^J4)VHRisO{;j z`N0+X+QW)Bj$E^s?hVFiDOkX%EnyzXU3{&VIku)UXQ3_%wVm{Qv#DiVGb6jUr{=MC z#puoUDtsQ2*i_E!elGE#f(t!T`kA&Wc6IV_84P@Nch!KC{|b}x;QYDe>8X%BUobQN z%h+_rRahSKY#zax?yT2gtJYyQb7U{b%FHV}+UTYeHA}W#5S(ZrhdI;eW<=l%ToIyu z5)WCHhpt(l6Sk;{Wbn!R-K{I2$`v=h9q9D_!RUMkyng1MKP~M4Q7G{D-+d~E;EkHE zS15CK2|S9wJHN)P;@_3Ws2oB!*c)livo7)_rNSi4Lil=h-)+}OE6-oR8yWmU*Y z>xWe;mH{5Cw{3Mw{P-=-;*d$9>Ub~S^}y^kWp5+eZwJclPKRIQeob*IRDPkdPKp3j zreE9C??Tb5l~e+&@^9#})i4bACQ#9`d^Ne+|Q=F z*5%fCvst*g%2G8@*R&!yfUTcgyyi<7MKS_#{OnaDJ<1A?LUCco%70F}|m}phM0ZTWP_Z{#oinO(V{5oN9$Wnx+$By}s&NZgtWj~48Xk>I-Y2}YDY#g5Dz zhC|I+iAPw+H_};Ql_bN9EdgE_XQM|+xD|ib!{6u~T?X5}p|_K)mE2K<6ulhBzP98@ zA^M{-#yt*6yNvVx0qVU*rzfhyv{z)V8P?VI>(t&Syx2sf#t zP}06b`i@%Nn)QRcP3q=om9CH4z7Y^R8{~O*^slGGsS{)4QxEjZedv+-BP{G;+8hsX zNd?ZK6ycra)_2yVhe8tTI)u9&DmS!_&2+aRyoDf*7%v7qu$k0!7oXw@)EDw9Zq0!V z^2TBEA*27SqG)q8L)$2lyJ)T&Fl}0q5WqI9s*o6IEEs_RD&`RYR6|2G=xb5u$4SGC z`{4=T8TNCq^pLOJJlms(marP-nhC4dohF^FilCv5M6QFE!-F~L69?wZiTxhhAEp`A z6F2%!CB2mP&8PoTZtO265o#WME7EjNDY9F`mnrxbXdo^2n>7sj@^8-ul1QI1F(ny< zUf}tZ@M+$s2cqtYTzudt#!HDmt=N%^haui;ibvkI$7N33X7~*C9!j@`aD*HKc7R*ImyPu!p zzYO(B)z(O}Id{HSKq5dAXP)YHKi3w&|AB z>h6~E_)zSnH7p^R?jhpq&^RGI!q0WzJ|t6!IlEvm*O7kG;#sgeRGj+xQkj+f?8FI| zCdfXt=jCGSF*Ur(URA&=C#{P%{Fz>hdQ<-~$NC9bqrpy@EOXO(H*Clki`xIos;YH= z|3U_NBLWz2V<_&`k;yooR_uSTGP^JW_MNd-LI9|gU;^fkA9n_Q+Mr{!F7_~UXXw(s zbLPI}xY;!i`N$IlkkFitJropOB1eqeNH~?!VgD+S%~VxL)7)`ew11N0wf%Y)@4pLxLQ&K`4UK z*PPyaAg8+ewS0sBforn=95+R8+8ei=e34w{ZVWS3*$N(aCkBQ#YrGlzQ`H!jRD7{L zz2#HzE@*X8IZN1_&&y&W`a|^V>FZqebC-*b&%~;cWfLgaD7=(=H7qKOwt8_E#Ntv4 z#->B_na*Y#63oIzXW#Da*^|Q!U>1oLJ&r;AB?s|Lfr)%VYZzzsFj;n}!S-}h&~WUz z%Byo{QeQZ8|Je?VFu5Z!NAl3RNHuWWVyx?ZP!L}a2Po#QH!`u{6_eM_{-(caEn4T% zyBR;ihGr!9{px!9``+m08#P47kkD}nJ|)!%>Rc(~ zov)F2MHX7*nhlrTI0_ZSfB*R-gjOer#h40O$J1584`a2uGa!4qengy^nn%><^xX{p z!z}-1z8|?90HR-TsQF)UA-?tTn>I%z!ME#kkHhE5CJV+=&Q?cq%{`*m?la7!gYa%p z=yHlAY9_pRs8=iU8NYcA1@Ynuh4QBjRuk7c-HP7xb)^n=Vah4ZX!pl~+|8jk0~3$& zMOkKP#AT?0yJW8Q=w*u z9{f2*+aqRPrDCfzBXP3m^ZnZ04-??8QMGyQx^GIO##O#rpbU2hCWE(cHoD;q1!d?g zOsF?(h~KrU_cDlWO{^2pN`E8^B7v_jgA*(g&jcw&kGcB?p>>lKHKWJb>__hGixH{E zG6~>N@%}lSwAL7_IH;NyfAgv)ADpRoi`?`xe!R(%zv{uNR^HcoyUyw>tm&2q4kNXm z;ug1U&@A}2+6lDuAb#|?cs+ckr&C4}QeO}NGn^9v-1M}dmXOk2#)87G;+(^3Xa{CK zmz3Y%mZ$J{BYJ9XzH`sA*y2{+ZPf(qk6hH<;J#duIZCJA_Sf+ViF9Xs$9zn)38Q0a z!poA?B+#*X#N}D^M zD=>0y3#oQoFOaC>Pex|=HL1^k<8SaI)%3h+d;HV+f<9;of=W2J@2ZS&LCC7a9@6nEGKku7spJ^>A!dVNky=Z!=GV<;xtB4l6VQ#4z zY&W{AYP_&u)_l`pui1Eude#t4mAfV0(MP@>7SqfUrJ3>V{KGCX`o;xkEy~QZx`8k( zk}#fS*2B!#VB$58Q^dp$rY+|WR<%3eWW8I2%xX>V?sY(Jo-;+jul5-|roH|^WDOb* zcg=Fa{z8p|tSmO(eNfJh8sxKqEdEK_3RyLW#Q0D*wCJL?mN1lEpMk68J={5X$p|)z zKff9ZTO)_QKJ25pHd*3+l1YMptY2s?Oq%~?M<}OtMSnE0!T9~2DRM)2h8^a+_^t^7 z5FoKU1G2On`P8+6P5Mt^1Jal4Cnx1-L6o25=>iS})l7)`RWWA1;#O#}o`kB9yMA~@ zqpwyWC_e4G$PZ(#gleKHR!OTu$+`R6*XEe|Z1BkiSNqmKt2&(_!E%0m+0W;ZJ|4{V znQKtSg&)eqWvE9N#p#7c%9n`SPFn%i7YO^kvwp_zf-4Iwy&@10#|=98rq+z%AqMlY zewOLBOQK{i3r}M|kDt%`N~bxcthUdGjM^h-_nK_u4bM$cGyg_m-f4s#>u7zwT)U6esG@-b%3eLj~W>VH}Sz^I^a)tHFuF zFVx0;Vw1chySWr&dmfXTF=DMTZ*v%*;iVr2f;87quX8838C3m$svw>AS0|gz`xp}* zn&AsZDkg`_XhQ^|%FXn49#t%UNJoxGNa47w4f4pES^J)A&~+2A$vl)m|My|g}&S+$Q!#}PJdpS*rrH+wDc@$m;%yRV8!)6qS~ znp}$XT1AT|9wm{5iG_QKcR%otBP$&${mF_(|Ge}+UkMxd?$~d~_IXfOV!G?ygvVv> z0RgYov?`!++^t2NK)g-UGXaM@fnRVi*dJhG4zO@9tIT;v8w+~*Ner7yfPxHp0y6_N zv_luSZYU9-H}nhFWT-a1kw!3;($MAz8l`tG-O zdwF{@q8k4wNtLxG5U63v*~((yVhCX+zzhs-M%%{t$grIg8J$yi>*~ ztKOaGE#{K1gV@_fE-R~`_j&vL!t)f*wHC&VQ?oQ&vIleNivvWdIMy?a8Ime`y$I^y%G-y{{Y^OmYXK6lClzK#sLIK*P;i)dBELmR@@|p0tcn zDr7!r;NmrOIIu=-wl_4obyq{w_HG=V?Z#sftIoP-4cPIQ1RThDzB|jPY;ktXGp<{t zb>m7EEs82#A%`-xAB~%GYc-AjGp09G%d<46@`I=lNga^EkhHMq^s+5SPb3* zet{om#sWOwy=_C$f46!hV!2GR%%4-D`qBKxlm}YmL`$9T)lXrhk{cpus;p)i(=1WM zsZyVVHf;=J9m}Gw^ESs^XO`rsHGya#TerEW$R6N`x@3%F{Vy%g$+1f6|x` zrJQV=@HJFp9i6yL!}kO`kN~CIEn2I5z-lAHEQ))glh)GsRbvHxGYeX9QbMgswQCbD zi_>*1W=!$t5S&U*S*n2nBDU)+X#@(j^xIN`$F#e^T5^Oenw^gWZgNS*re0`IaPwH= z6CO32ynR6u$9oTlTQ9NfqQ36}kc-BB@btUo%@lN)bNN+I?J7@UNB$~>R9WOBYq0NZ zvr+;|&R}R>dS|Vsf`P5A8J)GEyC(FuvKQa;JvNccpXZ;4-bX!-mQ_=9nUb32Yj@dw z`0CA! 0, Total - symsum(R(h), h, 0, height)); -coffer(height) = piecewise(height < 0, Total, height >= 0, coffer(height-1) - R(height)); -R(height) = coffer(height-1)/Total * IV; -syms x -r = R(x); - -fplot(r, [0, 5]); -fibonacci \ No newline at end of file diff --git a/lib/rlepluslazy/bitvec.go.bak b/lib/rlepluslazy/bitvec.go.bak deleted file mode 100644 index da713bc432d..00000000000 --- a/lib/rlepluslazy/bitvec.go.bak +++ /dev/null @@ -1,63 +0,0 @@ -package rlepluslazy - -type rbitvec struct { - index int - - bits uint64 - bitCap byte - - vec []byte -} - -func readBitvec(vec []byte) *rbitvec { - bv := &rbitvec{vec: vec} - for n := 7; n >= 0; n-- { - var o uint64 - if len(bv.vec) > n { - o = uint64(bv.vec[n]) - } - - bv.bits = bv.bits<<8 | o - bv.index++ - } - bv.bitCap = 64 - return bv -} - -const ( - minCap = 8 -) - -var bitMasks = [9]byte{ - 0x0, - 0x1, - 0x3, - 0x7, - 0xF, - 0x1F, - 0x3F, - 0x7F, - 0xFF, -} - -func (bv *rbitvec) Get(count byte) byte { - res := byte(bv.bits) & bitMasks[count] - bv.bits = bv.bits >> count - bv.bitCap = bv.bitCap - count - - if bv.bitCap < minCap { - var add uint64 - for n := 6; n >= 0; n-- { - var o uint64 - if len(bv.vec) > bv.index+n { - o = uint64(bv.vec[bv.index+n]) - } - - add = add<<8 | o - } - bv.index = bv.index + 7 - bv.bits = bv.bits | add<<(bv.bitCap) - bv.bitCap = bv.bitCap + 7*8 - } - return res -} diff --git a/lib/rlepluslazy/rleplus.go b/lib/rlepluslazy/rleplus.go index 6bf8f378968..100007d6539 100644 --- a/lib/rlepluslazy/rleplus.go +++ b/lib/rlepluslazy/rleplus.go @@ -45,150 +45,3 @@ func (rle *RLE) Count() (uint64, error) { } return Count(it) } - -/* - -type change struct { - set bool - reset bool - index uint64 -} -func (c change) valid() bool { - return c.reset || c.set -} - -func (rle *RLE) RunIterator() (RunIterator, error) { - if err != nil { - return nil, err - } - - if len(rle.changes) == 0 { - return source, nil - } - - ch := rle.changes - // using stable sort because we want to preserve change order - sort.SliceStable(ch, func(i int, j int) bool { - return ch[i].index < ch[j].index - }) - for i := range ch { - if i+1 >= len(ch) { - break - } - if ch[i].index == ch[i+1].index { - ch[i].set = false - ch[i].reset = false - } - } - - sets := make([]uint64, 0, len(ch)/2) - clears := make([]uint64, 0, len(ch)/2) - for _, c := range ch { - if c.set { - sets = append(sets, c.index) - } else if c.reset { - clears = append(clears, c.index) - } - } - - setRuns, err := RunsFromSlice(sets) - if err != nil { - return nil, err - } - clearRuns, err := RunsFromSlice(clears) - if err != nil { - return nil, err - } - - it := &chit{source: source, sets: setRuns, clears: clearRuns} - return nil, nil -} - -type chit struct { - source RunIterator - sets RunIterator - clears RunIterator - - next Run - - nextSource Run - nextSet Run - nextClear Run -} - -func (it *chit) prep() error { - var err error - if !it.nextSource.Valid() && it.source.HasNext() { - it.nextSource, err = it.source.NextRun() - if err != nil { - return err - } - } - - if !it.nextSet.Valid() && it.sets.HasNext() { - it.nextSet, err = it.sets.NextRun() - if err != nil { - return err - } - } - - if !it.nextClear.Valid() && it.clears.HasNext() { - it.nextClear, err = it.clears.NextRun() - if err != nil { - return err - } - } - - for len(it.changes) != 0 && !it.changes[0].valid() { - it.changes = it.changes[1:] - } - - var ch change - if len(it.changes) != 0 { - ch = it.changes[0] - } - - if it.source.HasNext() { - var err error - it.nextRun, err = it.source.NextRun() - if err != nil { - return err - } - } - - if ch.valid() && ch.index < it.runIndex+it.nextRun.Len { - if ch.set != it.nextRun.Val { - // split run - whole := it.nextRun - it.nextRun.Len = ch.index - it.runIndex - - } - - // if we are here then change was valid so len(it.changes) != 0 - it.changes = it.changes[1:] - } else { - it.runIndex = it.runIndex + it.nextRun.Len - } - return nil -} - -func (it *chit) HasNext() bool { - return it.nextRun.Valid() -} - -func (it *chit) NextRun() (Run, error) { - return it.nextRun, it.prep() -} - -func (rle *RLE) Set(index uint64) { - rle.changes = append(rle.changes, change{set: true, index: index}) -} - -func (rle *RLE) Clear(index uint64) { - rle.changes = append(rle.changes, change{reset: true, index: index}) -} - -func (rle *RLE) Merge(other *RLE) RunIterator { - return nil -} -*/ diff --git a/lib/rlepluslazy/rleplus_reader.go b/lib/rlepluslazy/rleplus_reader.go index 5fff2ccd63b..0b3a77da002 100644 --- a/lib/rlepluslazy/rleplus_reader.go +++ b/lib/rlepluslazy/rleplus_reader.go @@ -1,8 +1,7 @@ package rlepluslazy import ( - "encoding/binary" - + "github.com/multiformats/go-varint" "golang.org/x/xerrors" ) @@ -64,7 +63,12 @@ func (it *rleIterator) prep() error { return xerrors.Errorf("run too long: %w", ErrDecode) } } - it.nextRun.Len, _ = binary.Uvarint(buf) + var err error + it.nextRun.Len, _, err = varint.FromUvarint(buf) + if err != nil { + return err + } + } } diff --git a/lib/rlepluslazy/runs.go b/lib/rlepluslazy/runs.go index be2ed5bcc99..034a449c9bb 100644 --- a/lib/rlepluslazy/runs.go +++ b/lib/rlepluslazy/runs.go @@ -59,7 +59,7 @@ func (it *addIt) prep() error { return nil } - if !it.arun.Val && !it.brun.Val { + if !(it.arun.Val || it.brun.Val) { min := it.arun.Len if it.brun.Len < min { min = it.brun.Len