Lotus/fix all tests (#52)

* add keyinfo type for signing things
* add bitfield type
* update serialization code
* add method to create bigint from string
* messages use bytes for params
* make driver assertions give better errors
* add rleplus type
* add storage miner actor types
* add storage power actor types
* add code gen for all new types
* update miner test with new type system

extern/rleplus/internal/bitvector.go

@@ -0,0 +1,154 @@
+package bitvector
+
+import (
+	"errors"
+	"log"
+)
+
+var (
+	// ErrOutOfRange - the index passed is out of range for the BitVector
+	ErrOutOfRange = errors.New("index out of range")
+)
+
+// BitNumbering indicates the ordering of bits, either
+// least-significant bit in position 0, or most-significant bit
+// in position 0.
+//
+// It it used in 3 ways with BitVector:
+// 1. Ordering of bits within the Buf []byte structure
+// 2. What order to add bits when using Extend()
+// 3. What order to read bits when using Take()
+//
+// https://en.wikipedia.org/wiki/Bit_numbering
+type BitNumbering int
+
+const (
+	// LSB0 - bit ordering starts with the low-order bit
+	LSB0 BitNumbering = iota
+
+	// MSB0 - bit ordering starts with the high-order bit
+	MSB0
+)
+
+// BitVector is used to manipulate ordered collections of bits
+type BitVector struct {
+	Buf []byte
+
+	// BytePacking is the bit ordering within bytes
+	BytePacking BitNumbering
+
+	// Len is the logical number of bits in the vector.
+	// The last byte in Buf may have undefined bits if Len is not a multiple of 8
+	Len uint
+}
+
+// NewBitVector constructs a new BitVector from a slice of bytes.
+//
+// The bytePacking parameter is required to know how to interpret the bit ordering within the bytes.
+func NewBitVector(buf []byte, bytePacking BitNumbering) *BitVector {
+	return &BitVector{
+		BytePacking: bytePacking,
+		Buf:         buf,
+		Len:         uint(len(buf) * 8),
+	}
+}
+
+// Push adds a single bit to the BitVector.
+//
+// Although it takes a byte, only the low-order bit is used, so just use 0 or 1.
+func (v *BitVector) Push(val byte) {
+	if v.Len%8 == 0 {
+		v.Buf = append(v.Buf, 0)
+	}
+	lastIdx := v.Len / 8
+
+	switch v.BytePacking {
+	case LSB0:
+		v.Buf[lastIdx] |= (val & 1) << (v.Len % 8)
+	default:
+		v.Buf[lastIdx] |= (val & 1) << (7 - (v.Len % 8))
+	}
+
+	v.Len++
+}
+
+// Get returns a single bit as a byte -- either 0 or 1
+func (v *BitVector) Get(idx uint) (byte, error) {
+	if idx >= v.Len {
+		return 0, ErrOutOfRange
+	}
+	blockIdx := idx / 8
+
+	switch v.BytePacking {
+	case LSB0:
+		return v.Buf[blockIdx] >> (idx % 8) & 1, nil
+	default:
+		return v.Buf[blockIdx] >> (7 - idx%8) & 1, nil
+	}
+}
+
+// Extend adds up to 8 bits to the receiver
+//
+// Given a byte b == 0b11010101
+// v.Extend(b, 4, LSB0) would add < 1, 0, 1, 0 >
+// v.Extend(b, 4, MSB0) would add < 1, 1, 0, 1 >
+//
+// Panics if count is out of range
+func (v *BitVector) Extend(val byte, count uint, order BitNumbering) {
+	if count > 8 {
+		log.Panicf("invalid count")
+	}
+
+	for i := uint(0); i < count; i++ {
+		switch order {
+		case LSB0:
+			v.Push((val >> i) & 1)
+		default:
+			v.Push((val >> (7 - i)) & 1)
+		}
+	}
+}
+
+// Take reads up to 8 bits at the given index.
+//
+// Given a BitVector < 1, 1, 0, 1, 0, 1, 0, 1 >
+// v.Take(0, 4, LSB0) would return 0b00001011
+// v.Take(0, 4, MSB0) would return 0b11010000
+//
+// Panics if count is out of range
+func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) {
+	if count > 8 {
+		log.Panicf("invalid count")
+	}
+
+	for i := uint(0); i < count; i++ {
+		val, _ := v.Get(index + i)
+
+		switch order {
+		case LSB0:
+			out |= val << i
+		default:
+			out |= val << (7 - i)
+		}
+	}
+	return
+}
+
+// Iterator returns a function, which when invoked, returns the number
+// of bits requested, and increments an internal cursor.
+//
+// When the end of the BitVector is reached, it returns zeroes indefinitely
+//
+// Panics if count is out of range
+func (v *BitVector) Iterator(order BitNumbering) func(uint) byte {
+	cursor := uint(0)
+	return func(count uint) (out byte) {
+		if count > 8 {
+			log.Panicf("invalid count")
+		}
+
+		out = v.Take(cursor, count, order)
+		cursor += count
+		return
+	}
+}

extern/rleplus/rleplus.go

@@ -0,0 +1,204 @@
+package rleplus
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"sort"
+
+	bitvector "github.com/filecoin-project/chain-validation/extern/rleplus/internal"
+)
+
+// Version is the 2 lowest bits of this constant
+const Version = 0
+
+var (
+	// ErrRunLengthTooLarge - data implies a run-length which isn't supported
+	ErrRunLengthTooLarge = fmt.Errorf("run length too large for RLE+ version %d", Version)
+
+	// ErrDecode - invalid encoding for this version
+	ErrDecode = fmt.Errorf("invalid encoding for RLE+ version %d", Version)
+
+	// ErrWrongVersion - wrong version of RLE+
+	ErrWrongVersion = errors.New("invalid RLE+ version")
+)
+
+// Encode returns the RLE+ representation of the provided integers.
+// Also returned is the number of bits required by this encoding,
+// which is not necessarily on a byte boundary.
+//
+// The RLE+ spec is here: https://github.com/filecoin-project/specs/blob/master/data-structures.md#rle-bitset-encoding
+// and is described by the BNF Grammar:
+//
+//    <encoding> ::= <header> <blocks>
+//    <header> ::= <version> <bit>
+//    <version> ::= "00"
+//    <blocks> ::= <block> <blocks> | ""
+//    <block> ::= <block_single> | <block_short> | <block_long>
+//    <block_single> ::= "1"
+//    <block_short> ::= "01" <bit> <bit> <bit> <bit>
+//    <block_long> ::= "00" <unsigned_varint>
+//    <bit> ::= "0" | "1"
+//
+// Filecoin specific:
+// The encoding is returned as a []byte, each byte packed starting with the low-order bit (LSB0)
+func Encode(ints []uint64) ([]byte, uint, error) {
+	v := bitvector.BitVector{BytePacking: bitvector.LSB0}
+	firstBit, runs := RunLengths(ints)
+
+	// Add version header
+	v.Extend(Version, 2, bitvector.LSB0)
+
+	v.Push(firstBit)
+
+	for _, run := range runs {
+		switch {
+		case run == 1:
+			v.Push(1)
+		case run < 16:
+			v.Push(0)
+			v.Push(1)
+			v.Extend(byte(run), 4, bitvector.LSB0)
+		case run >= 16:
+			v.Push(0)
+			v.Push(0)
+			// 10 bytes needed to encode MaxUint64
+			buf := make([]byte, 10)
+			numBytes := binary.PutUvarint(buf, run)
+			for i := 0; i < numBytes; i++ {
+				v.Extend(buf[i], 8, bitvector.LSB0)
+			}
+		default:
+			return nil, 0, ErrRunLengthTooLarge
+		}
+	}
+
+	return v.Buf, v.Len, nil
+}
+
+// Decode returns integers represented by the given RLE+ encoding
+//
+// The length of the encoding is not specified.  It is inferred by
+// reading zeroes from the (possibly depleted) BitVector, by virtue
+// of the behavior of BitVector.Take() returning 0 when the end of
+// the BitVector has been reached. This has the downside of not
+// being able to detect corrupt encodings.
+//
+// The passed []byte should be packed in LSB0 bit numbering
+func Decode(buf []byte) (ints []uint64, err error) {
+	if len(buf) == 0 {
+		return
+	}
+
+	v := bitvector.NewBitVector(buf, bitvector.LSB0)
+	take := v.Iterator(bitvector.LSB0)
+
+	// Read version and check
+	// Version check
+	ver := take(2)
+	if ver != Version {
+		return nil, ErrWrongVersion
+	}
+
+	curIdx := uint64(0)
+	curBit := take(1)
+	var runLength int
+	done := false
+
+	for done == false {
+		y := take(1)
+		switch y {
+		case 1:
+			runLength = 1
+		case 0:
+			val := take(1)
+
+			if val == 1 {
+				// short block
+				runLength = int(take(4))
+			} else {
+				// long block
+				var buf []byte
+				for {
+					b := take(8)
+					buf = append(buf, b)
+
+					if b&0x80 == 0 {
+						break
+					}
+
+					// 10 bytes is required to store math.MaxUint64 in a uvarint
+					if len(buf) > 10 {
+						return nil, ErrDecode
+					}
+				}
+				x, _ := binary.Uvarint(buf)
+
+				if x == 0 {
+					done = true
+				}
+				runLength = int(x)
+			}
+		}
+
+		if curBit == 1 {
+			for j := 0; j < runLength; j++ {
+				ints = append(ints, curIdx+uint64(j))
+			}
+		}
+		curIdx += uint64(runLength)
+		curBit = 1 - curBit
+	}
+
+	return
+}
+
+// RunLengths transforms integers into its bit-set-run-length representation.
+//
+// A set of unsigned integers { 0, 2, 4, 5, 6 } can be thought of as
+// indices into a bitset { 1, 0, 1, 0, 1, 1, 1 } where bitset[index] == 1.
+//
+// The bit set run lengths of this set would then be { 1, 1, 1, 1, 3 },
+// representing lengths of runs alternating between 1 and 0, starting
+// with a first bit of 1.
+//
+// Duplicated numbers are ignored.
+//
+// This is a helper function for Encode()
+func RunLengths(ints []uint64) (firstBit byte, runs []uint64) {
+	if len(ints) == 0 {
+		return
+	}
+
+	// Sort our incoming numbers
+	sort.Slice(ints, func(i, j int) bool { return ints[i] < ints[j] })
+
+	prev := ints[0]
+
+	// Initialize our return value
+	if prev == 0 {
+		firstBit = 1
+	}
+
+	if firstBit == 0 {
+		// first run of zeroes
+		runs = append(runs, prev)
+	}
+	runs = append(runs, 1)
+
+	for _, cur := range ints[1:] {
+		delta := cur - prev
+		switch {
+		case delta == 1:
+			runs[len(runs)-1]++
+		case delta > 1:
+			// add run of zeroes if there is a gap
+			runs = append(runs, delta-1)
+			runs = append(runs, 1)
+		default:
+			// repeated number?
+		}
+		prev = cur
+	}
+	return
+}